Allow utilities.yaml_load to read extended YAML

Address issue #49

src/fmu/config/_configparserfmu_ipl.py

@@ -393,7 +393,6 @@ def _freeform_handle_entry(
 
     # inner function
     def _fixtheentry(variable, myval, subtype, count=None, template=False):
-
         logger.info("Fix freeform entry %s (subtype %s)", variable, subtype)
         tmpvalue = str(myval)
         if "~" in tmpvalue:

src/fmu/config/configparserfmu.py

@@ -388,13 +388,11 @@ def _cleanify_doubleunderscores(self):
         newcfg = deepcopy(self._config)
 
         for key, val in self._config.items():
-
             if isinstance(val, dict):
                 subkeyorder = []
                 _tmps = {}
 
                 for subkey, subval in val.items():
-
                     if subkey.startswith("__"):
                         if isinstance(subval, dict):
                             for subsubkey, subsubval in subval.items():
@@ -507,7 +505,6 @@ def _get_sysinfo(commentmarker="#"):
 
     @staticmethod
     def _force_create_folders(folderlist):
-
         for folder in folderlist:
             if folder is None:
                 continue
@@ -519,7 +516,6 @@ def _force_create_folders(folderlist):
 
     @staticmethod
     def _check_folders(folderlist):
-
         for folder in folderlist:
             if folder is None:
                 continue
@@ -594,7 +590,6 @@ def _get_required_form(stream, template=False, ipl=False):
         if isinstance(stream, list):
             pass
         elif isinstance(stream, str):
-
             if "~" in stream:
                 value, tvalue = stream.split("~")
                 value = value.strip()

src/fmu/config/etc.py

@@ -64,7 +64,6 @@ class Interaction(object):
     """
 
     def __init__(self):
-
         self._callclass = None
         self._caller = None
         self._lformat = None
@@ -311,7 +310,6 @@ def _get_class_from_frame(frame):
         return outer[0]
 
     def _output(self, idx, level, string):
-
         # pylint: disable=too-many-branches
 
         prefix = ""

src/fmu/config/fmuconfigrunner.py

@@ -21,7 +21,6 @@
 
 
 def _do_parse_args(args):
-
     if args is None:
         args = sys.argv[1:]
 

src/fmu/config/utilities.py

@@ -1,40 +1,57 @@
 """Module with some simple functions, e.g. for parsing for YAML into RMS
 """
 
+from yaml.loader import Loader
+
 # for ordered dicts!
 from fmu.config import oyaml as yaml
+from fmu.config._loader import ConstructorError, FmuLoader
 
 
-def yaml_load(filename, safe=True, tool=None):
+def yaml_load(filename, safe=True, tool=None, loader="standard"):
     """Load as YAML file, return a dictionary of type OrderedDict which is the config.
 
     Returning an ordered dictionary is a main feature of this loader. It makes it much
-    easier to compare the dictionaries returned.
+    easier to compare the dictionaries returned. In addition, it allows for reading the
+    input (extended) YAML format, if key ``allow_extended`` is True.
 
     Args:
         filename (str): Name of file (YAML formatted)
-        safe (bool): If True (default), then use `safe_load`
+        safe (bool): If True (default), then use `safe_load` when allow_extended is
+            set to False. Not applied if loader is "fmu".
         tool (str): Refers to a particular main section in the config.
-            Default is None, which measn 'all'.
+            Default is None, which means 'all'.
+        loader (str): If "fmu", the in-house FMU extended YAML loader that allows
+            use of e.g. `!include` is applied; otherwise the default is "standard" YAML.
 
     Example::
         >>> import fmu.config.utilities as utils
         >>> cfg = utils.yaml_load('somefile.yml')
 
     """
 
+    useloader = FmuLoader if loader.lower() == "fmu" else Loader
+
     with open(filename, "r", encoding="utf-8") as stream:
-        if safe:
-            cfg = yaml.safe_load(stream)
-        else:
-            cfg = yaml.load(stream)
+        try:
+            if safe and loader.lower() != "fmu":
+                cfg = yaml.safe_load(stream)
+            else:
+                cfg = yaml.load(stream, Loader=useloader)
+        except ConstructorError as cerr:
+            if "!include" in str(cerr):
+                print(
+                    "\n*** Consider setting loader='fmu' to read fmu.config "
+                    "input style ***\n"
+                )
+            raise
 
     if tool is not None:
         try:
             newcfg = cfg[tool]
             cfg = newcfg
         except Exception as exc:  # pylint: disable=broad-except
-            print("Cannot import: {}".format(exc))
+            print(f"Cannot import: {exc}")
             return None
 
     return cfg

tests/data/yml/drogon/input/_access.yml

@@ -0,0 +1,5 @@
+asset:
+  name: Drogon
+ssdl:
+  access_level: internal
+  rep_include: true

tests/data/yml/drogon/input/_auxiliary.yml

@@ -0,0 +1,79 @@
+# This is auxiliary metadata that contains helper setup for mostly for SUMO
+# * aliases
+# * display settings
+
+# The key names are the modelling names used in RMS project (e.g. TopVolantis) while the "name"
+# shall be the official SMDA name
+
+MSL:
+  stratigraphic: False
+  name: MSL
+  display_name: MSL
+
+Seabase:
+  stratigraphic: False
+  name: Seabase
+  display_name: Seabase
+
+TopVolantis:
+  stratigraphic: True
+  name: VOLANTIS GP. Top  # this is the official DB name
+  display_name: Top Volantis
+  # additional names, given for demo purpose:
+  alias:
+    - TopVOLANTIS
+    - TOP_VOLANTIS
+  straigraphic_alias:
+    - TopValysar
+    - Valysar Fm. Top
+
+TopTherys:
+  stratigraphic: True
+  name: Therys Fm. Top
+  display_name: Top Therys
+
+TopVolon:
+  stratigraphic: True
+  name: Volon Fm. Top
+  display_name: Top Volon
+
+BaseVolon:
+  stratigraphic: True
+  name: Volon Fm. Base
+  display_name: Base Volon
+
+BaseVolantis:
+  stratigraphic: True
+  name: VOLANTIS GP. Base
+  display_name: Base Volantis
+
+Mantle:
+  stratigraphic: False
+  name: Mantle
+
+#=========================================================================================
+# zones
+Above:
+  stratigraphic: False
+  name: Above
+  display_name: Above Reservoir
+
+Valysar:
+  stratigraphic: True
+  name: Valysar Fm.
+  display_name: Valysar
+
+Therys:
+  stratigraphic: True
+  name: Therys Fm.
+  display_name: Therys
+
+Volon:
+  stratigraphic: True
+  name: Volon Fm.
+  display_name: Volon
+
+Below:
+  stratigraphic: False
+  name: Below
+  display_name: Below Reservoir

tests/data/yml/drogon/input/_dates.yml

@@ -0,0 +1,19 @@
+  SEISMIC_HIST_DATES: &4d_dates
+    - 2018-01-01
+    - 2018-07-01
+    - 2019-07-01
+    - 2020-07-01
+
+  SEISMIC_HIST_DIFFDATES: &4d_diffdates
+    - [2018-07-01, 2018-01-01]
+    - [2019-07-01, 2018-01-01]
+    - [2020-07-01, 2018-01-01]
+    - [2019-07-01, 2018-07-01]
+    - [2020-07-01, 2019-07-01]
+
+  ECLIPSE_INIT_DATE: &ecl_init_date 2018-01-01 #init date
+
+  # eclipse dates should be aligned with rmsevents IORESTART keyword (restart reporting)
+  ECLIPSE_HIST_DATES: *4d_dates  # same dates as in SEISMIC_HIST_DATES
+
+  ECLIPSE_HIST_DIFFDATES: *4d_diffdates  # same dates as in SEISMIC_HIST_DIFFDATES

tests/data/yml/drogon/input/_dates_pred.yml

@@ -0,0 +1,20 @@
+  #seismic pred dates should include diff dates and relevant single dates
+  SEISMIC_PRED_DATES: &4d_pred_dates
+    - 2018-01-01
+    - 2020-07-01
+    - 2025-01-01
+
+  SEISMIC_PRED_DIFFDATES: &4d_pred_diffdates
+    - [2025-01-01, 2018-01-01]
+    - [2025-01-01, 2020-07-01]
+
+  #ecl_dates should be aligned with rmsevents IORESTART keyword (restart reporting)
+  ECLIPSE_INIT_DATES: &ecl_init_dates
+    - 2018-01-01
+
+  ECLIPSE_PRED_DATES: &ecl_pred_dates
+    - 2025-01-01
+
+  ECLIPSE_PRED_DIFFDATES: &ecl_pred_diffdates
+    - [2025-01-01, 2018-01-01]
+    - [2025-01-01, 2020-07-01]

tests/data/yml/drogon/input/_fwl_2dtable.yml

@@ -0,0 +1,13 @@
+# These data are currently *not* in use ; they are here for demo purpose!
+# This is primarely an example on how to read 2D data from YAML
+# in the RMS project it will be a demo_*.py script to show how
+# it can be read by python
+
+#    ZONE1 VALYSAR              ZONE2 THERYS ZONE3        ZONE3 VOLON ZONE3
+- [1660 ~ <FWL_VALYSAR_REG1>, 1660 ~ <FWL_THERYS_REG1>, 1660 ~ <FWL_VOLON_REG1>]  # region 1 WestLowland
+- [1667 ~ <FWL_VALYSAR_REG2>, 1667 ~ <FWL_THERYS_REG2>, 1680 ~ <FWL_VOLON_REG2>]  # region 2 CentralSouth
+- [1668 ~ <FWL_VALYSAR_REG3>, 1667 ~ <FWL_THERYS_REG3>, 1680 ~ <FWL_VOLON_REG3>]  # region 3 CentralNorth
+- [1661 ~ <FWL_VALYSAR_REG4>, 1667 ~ <FWL_THERYS_REG4>, 1680 ~ <FWL_VOLON_REG4>]  # region 4 NorthHorst
+- [1655 ~ <FWL_VALYSAR_REG5>, 1664 ~ <FWL_THERYS_REG5>, 1680 ~ <FWL_VOLON_REG5>]  # region 5 CentralRamp
+- [1632 ~ <FWL_VALYSAR_REG6>, 1667 ~ <FWL_THERYS_REG6>, 1680 ~ <FWL_VOLON_REG6>]  # region 6 CentralHorst
+- [1688 ~ <FWL_VALYSAR_REG7>, 1667 ~ <FWL_THERYS_REG7>, 1690 ~ <FWL_VOLON_REG7>]  # region 7 EastLowland

tests/data/yml/drogon/input/_fwl_dict.yml

@@ -0,0 +1,44 @@
+# These data are currently *not* in use ; they are here for demo purpose!
+# This is primarely an example on how to read 2D data from YAML
+# in the RMS project it will be a demo_*.py script to show how
+# it can be read by python
+
+#    ZONE1 VALYSAR              ZONE2 THERYS ZONE3        ZONE3 VOLON ZONE3
+# - [1660 ~ <FWL_VALYSAR_REG1>, 1660 ~ <FWL_THERYS_REG1>, 1660 ~ <FWL_VOLON_REG1>]  # region 1 WestLowland
+# - [1667 ~ <FWL_VALYSAR_REG2>, 1667 ~ <FWL_THERYS_REG2>, 1680 ~ <FWL_VOLON_REG2>]  # region 2 CentralSouth
+# - [1668 ~ <FWL_VALYSAR_REG3>, 1667 ~ <FWL_THERYS_REG3>, 1680 ~ <FWL_VOLON_REG3>]  # region 3 CentralNorth
+# - [1661 ~ <FWL_VALYSAR_REG4>, 1667 ~ <FWL_THERYS_REG4>, 1680 ~ <FWL_VOLON_REG4>]  # region 4 NorthHorst
+# - [1655 ~ <FWL_VALYSAR_REG5>, 1664 ~ <FWL_THERYS_REG5>, 1680 ~ <FWL_VOLON_REG5>]  # region 5 CentralRamp
+# - [1632 ~ <FWL_VALYSAR_REG6>, 1667 ~ <FWL_THERYS_REG6>, 1680 ~ <FWL_VOLON_REG6>]  # region 6 CentralHorst
+# - [1688 ~ <FWL_VALYSAR_REG7>, 1667 ~ <FWL_THERYS_REG7>, 1690 ~ <FWL_VOLON_REG7>]  # region 7 EastLowland
+
+# better! use dictionary
+
+Westlowland:
+  Valysar: 1660 ~ <FWL_VALYSAR_REG1>
+  Therys: 1660 ~ <FWL_THERYS_REG1>
+  Volon: 1660 ~ <FWL_VOLON_REG1>
+Centralsouth:
+  Valysar: 1667 ~ <FWL_VALYSAR_REG2>
+  Therys: 1667 ~ <FWL_THERYS_REG2>
+  Volon: 1680 ~ <FWL_VOLON_REG2>
+CentralNorth:
+  Valysar: 1668 ~ <FWL_VALYSAR_REG3>
+  Therys: 1667 ~ <FWL_THERYS_REG3>
+  Volon: 1680 ~ <FWL_VOLON_REG3>
+NorthHorst:
+  Valysar: 1661 ~ <FWL_VALYSAR_REG4>
+  Therys: 1667 ~ <FWL_THERYS_REG4>
+  Volon: 1680 ~ <FWL_VOLON_REG4>
+CentralRamp:
+  Valysar: 1655 ~ <FWL_VALYSAR_REG5>
+  Therys: 1664 ~ <FWL_THERYS_REG5>
+  Volon: 1680 ~ <FWL_VOLON_REG5>
+CentralHorst:
+  Valysar: 1632 ~ <FWL_VALYSAR_REG6>
+  Therys: 1667 ~ <FWL_THERYS_REG6>
+  Volon: 1680 ~ <FWL_VOLON_REG6>
+Eastlowland:
+  Valysar: 1688 ~ <FWL_VALYSAR_REG7>
+  Therys: 1667 ~ <FWL_THERYS_REG7>
+  Volon: 1690 ~ <FWL_VOLON_REG7>

tests/data/yml/drogon/input/_masterdata.yml

@@ -0,0 +1,22 @@
+#========================================================================================
+# Master data section, to be used for SUMO master data metadata
+# In global_master, use:
+# masterdata: !include _masterdata.yml
+#========================================================================================
+
+smda:
+  country:
+    - identifier: Norway
+      uuid: ad214d85-8a1d-19da-e053-c918a4889309
+  discovery:
+    - short_identifier: DROGON
+      uuid: ad214d85-8a1d-19da-e053-c918a4889309
+  field:
+    - identifier: DROGON
+      uuid: 00000000-0000-0000-0000-000000000000
+  coordinate_system:
+    identifier: ST_WGS84_UTM37N_P32637
+    uuid: ad214d85-dac7-19da-e053-c918a4889309
+  stratigraphic_column:
+    identifier: DROGON_HAS_NO_STRATCOLUMN
+    uuid: 00000000-0000-0000-0000-000000000000