install docs, service definitions move

bycon/lib/beacon_response_generation.py

@@ -9,7 +9,7 @@
 from handover_generation import dataset_response_add_handovers
 from query_execution import ByconDatasetResults # execute_bycon_queries
 from query_generation import ByconQuery
-from read_specs import datasets_update_latest_stats, load_yaml_empty_fallback
+from read_specs import load_yaml_empty_fallback
 from response_remapping import *
 from variant_mapping import ByconVariant
 from schema_parsing import get_schema_file_path, object_instance_from_schema_name
@@ -612,22 +612,42 @@ def populatedCollections(self):
     def __collections_return_datasets(self):
         if not "dataset" in self.response_entity_id:
             return
-        dbstats = datasets_update_latest_stats()
-        for i, d_s in enumerate(dbstats):
-            ds_id = d_s.get("id", "___none___")
-            if ds_id in BYC["BYC_DATASET_IDS"]:
-                # TODO: remove verifier hack
-                for t in ["createDateTime", "updateDateTime"]:
-                    d = str(d_s.get(t, "1967-11-11"))
-                    if re.match(r'^\d\d\d\d\-\d\d\-\d\d$', d):
-                        dbstats[i].update({t:f'{d}T00:00:00+00:00'})
-
+        dbstats = __datasets_update_latest_stats()
         self.collections = dbstats
         self.collections_queries.update({"datasets":{}})
 
         return
 
 
+    # -------------------------------------------------------------------------#
+
+    def __datasets_update_latest_stats():
+        results = []
+        stat = []
+
+        stats = MongoClient(host=DB_MONGOHOST)[HOUSEKEEPING_DB][HOUSEKEEPING_INFO_COLL].find( { }, { "_id": 0 } ).sort( {"date": -1} ).limit( 1 )
+        stats = list(stats)
+
+        if len(stats) > 0:
+            stat = stats[0]
+        else:
+            return results
+        for coll_id, coll in BYC["dataset_definitions"].items():
+            if not coll_id in BYC.get("BYC_DATASET_IDS", []):
+                continue
+            if "datasets" in stat:
+                if (ds_vs := stat[ "datasets" ].get(coll_id)):
+                    if "filtering_terms" in BYC["response_entity_id"]:
+                        coll.update({ "filtering_terms": ds_vs.get("filtering_terms", []) } )
+            # TODO: remove verifier hack
+            for t in ["createDateTime", "updateDateTime"]:
+                d = str(d_s.get(t, "1967-11-11"))
+                if re.match(r'^\d\d\d\d\-\d\d\-\d\d$', d):
+                    coll.update({t:f'{d}T00:00:00+00:00'})
+            results.append(coll)
+        return results
+
+
     # -------------------------------------------------------------------------#
 
     def __collections_return_cohorts(self):

bycon/lib/read_specs.py

@@ -3,7 +3,6 @@
 from json_ref_dict import RefDict, materialize
 from os import path, pardir, scandir, environ
 from pathlib import Path
-from pymongo import MongoClient
 
 from cgi_parsing import prdbug
 from config import *
@@ -32,6 +31,13 @@ def read_service_definition_files():
             o = yaml.load( od , Loader=yaml.FullLoader)
         BYC.update({d: o})
 
+    e_d = always_merger.merge(
+        BYC.get("entity_defaults", {}),
+        BYC.get("services_entity_defaults", {})
+    )
+
+    BYC.update({"entity_defaults": e_d})
+
 
 ################################################################################
 
@@ -42,14 +48,11 @@ def update_rootpars_from_local_or_HOST():
         return
 
     p_c_p.append(LOC_PATH)
-    s_f = path.join(LOC_PATH, 'services_entity_defaults.yaml')
-    s = load_yaml_empty_fallback(s_f)
-    BYC.update({"entity_defaults": always_merger.merge(BYC.get("entity_defaults", {}), s)})
 
     # overwriting installation-wide defaults with instance-specific ones
     # _i.e._ matching the current domain (to allow presentation of different
     # Beacon instances from the same server)
-    i_ovr_f = path.join(LOC_PATH, "instance_overrides.yaml")
+    i_ovr_f = path.join(LOC_PATH, "instance_definitions.yaml")
     i_ovr = load_yaml_empty_fallback(i_ovr_f)
 
     if "local" in i_ovr:
@@ -81,46 +84,14 @@ def update_rootpars_from_local_or_HOST():
     return
 
 
-################################################################################
-
-def dbstats_return_latest():
-    # TODO: This is too hacky & should be moved to an external function
-    # which updates the database_definitions / beacon_info yamls...
-    stats = MongoClient(host=DB_MONGOHOST)[HOUSEKEEPING_DB][HOUSEKEEPING_INFO_COLL].find( { }, { "_id": 0 } ).sort( {"date": -1} ).limit( 1 )
-    stats = list(stats)
-    if len(stats) > 0:
-        return stats[0]
-    else:
-        return []
-
-
-################################################################################
-
-def datasets_update_latest_stats():
-    results = []
-    stat = dbstats_return_latest()
-    for coll_id, coll in BYC["dataset_definitions"].items():
-        if not coll_id in BYC.get("BYC_DATASET_IDS", []):
-            continue
-        if "datasets" in stat:
-            if (ds_vs := stat[ "datasets" ].get(coll_id)):
-                if "filtering_terms" in BYC["response_entity_id"]:
-                    coll.update({ "filtering_terms": ds_vs.get("filtering_terms", []) } )
-        # TODO: Add stats ...
-        results.append(coll)
-    return results
-
-
 ################################################################################
 
 def load_yaml_empty_fallback(yp):
     y = {}
     try:
-        # print(yp)
         with open( yp ) as yd:
             y = yaml.load( yd , Loader=yaml.FullLoader)
     except Exception as e:
-        # print(e)
         pass
     return y