Clean-up commodity group processing and storing

etsap-TIMES · Feb 19, 2024 · d4c2a11 · d4c2a11
1 parent 022cee2
commit d4c2a11
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 12 deletions.
diff --git a/xl2times/config/times_mapping.txt b/xl2times/config/times_mapping.txt
@@ -3,7 +3,7 @@ ALL_TS[ALL_TS] = TimeSlices(TS)
 B[DATAYEAR,VALUE] = TimePeriods(Year,B)
 COM[COM] = Commodities(Commodity)
 COM_DESC[REG,COM,TEXT] = Commodities(Region,Commodity,Description)
-COM_GMAP[REG,COM_GRP,COM] = CommodityGroupMap(Region,CommodityGroup,Commodity)
+COM_GMAP[REG,COM_GRP,COM] = CommodityGroups(Region,CommodityGroup,Commodity)
 COM_GRP[COM_GRP] = CommodityGroups(CommodityGroup)
 COM_LIM[REG,COM,BD] = Commodities(Region,Commodity,LimType)
 COM_PEAK[REG,COM_GRP] = Attributes(Region,Commodity,Attribute:COM_PEAK,VALUE:1)

diff --git a/xl2times/datatypes.py b/xl2times/datatypes.py
@@ -148,7 +148,6 @@ class TimesModel:
     all_regions: Set[str] = field(default_factory=set)
     processes: DataFrame = field(default_factory=DataFrame)
     commodities: DataFrame = field(default_factory=DataFrame)
-    com_gmap: DataFrame = field(default_factory=DataFrame)
     commodity_groups: DataFrame = field(default_factory=DataFrame)
     topology: DataFrame = field(default_factory=DataFrame)
     trade: DataFrame = field(default_factory=DataFrame)

diff --git a/xl2times/transforms.py b/xl2times/transforms.py
@@ -982,7 +982,6 @@ def complete_dictionary(
         "Attributes": model.attributes,
         "Commodities": model.commodities,
         "CommodityGroups": model.commodity_groups,
-        "CommodityGroupMap": model.com_gmap,
         "Processes": model.processes,
         "Topology": model.topology,
         "Trade": model.trade,
@@ -1177,7 +1176,6 @@ def name_comm_group(df):
     i = comm_groups["commoditygroup"] != comm_groups["commodity"]
 
     model.topology = comm_groups
-    model.com_gmap = comm_groups.loc[i, ["region", "commoditygroup", "commodity"]]
 
     return tables
 
@@ -1247,14 +1245,23 @@ def complete_commodity_groups(
     Complete the list of commodity groups
     """
 
-    commodities = generate_topology_dictionary(tables, model)[
-        "commodities_by_name"
-    ].rename(columns={"commodity": "commoditygroup"})
-    cgs_in_top = model.topology["commoditygroup"].to_frame()
-    commodity_groups = pd.concat([commodities, cgs_in_top])
-    model.commodity_groups = commodity_groups.drop_duplicates(
-        keep="first"
-    ).reset_index()
+    # Single member CGs i.e., CG and commodity are the same
+    single_cgs = (
+        model.commodities[["region", "commodity"]]
+        .drop_duplicates(ignore_index=True)
+        .copy()
+    )
+    single_cgs["commoditygroup"] = single_cgs["commodity"]
+    # Commodity groups from topology
+    top_cgs = (
+        model.topology[["region", "commodity", "commoditygroup"]]
+        .drop_duplicates(ignore_index=True)
+        .copy()
+    )
+    commodity_groups = pd.concat([single_cgs, top_cgs], ignore_index=True)
+    model.commodity_groups = commodity_groups.dropna().drop_duplicates(
+        ignore_index=True
+    )
 
     return tables