From c42d74356862e7cf2fedf6d089da867dacc10011 Mon Sep 17 00:00:00 2001
From: Olexandr Balyk <ob@facilitate.energy>
Date: Sat, 4 Jan 2025 15:51:06 -0500
Subject: [PATCH] Process comagg as well

---
 xl2times/__main__.py            |  2 +-
 xl2times/config/times-info.json |  4 ++--
 xl2times/config/veda-tags.json  | 26 +++++++++++++++++++++++++-
 xl2times/transforms.py          | 31 ++++++++++++++++++++++---------
 4 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/xl2times/__main__.py b/xl2times/__main__.py
index 79763664..34a20183 100644
--- a/xl2times/__main__.py
+++ b/xl2times/__main__.py
@@ -118,7 +118,7 @@ def convert_xl_to_times(
         transforms.capitalise_table_values,
         transforms.process_regions,
         transforms.process_commodities,
-        transforms.process_commodity_emissions,
+        transforms.convert_com_tables,
         transforms.process_time_periods,
         transforms.remove_exreg_cols,
         transforms.generate_dummy_processes,
diff --git a/xl2times/config/times-info.json b/xl2times/config/times-info.json
index 23f3dbf9..3ce803d9 100644
--- a/xl2times/config/times-info.json
+++ b/xl2times/config/times-info.json
@@ -775,13 +775,13 @@
       "REG",
       "YEAR",
       "COM",
-      "COM"
+      "COM2"
     ],
     "mapping": [
       "region",
       "year",
       "commodity",
-      "commodity"
+      "other_indexes"
     ]
   },
   {
diff --git a/xl2times/config/veda-tags.json b/xl2times/config/veda-tags.json
index ba82b3eb..ae550bee 100644
--- a/xl2times/config/veda-tags.json
+++ b/xl2times/config/veda-tags.json
@@ -33,6 +33,30 @@
       }
     ]
   },
+  {
+    "tag_name": "comagg",
+    "tag_allowed_in": [
+      "BY",
+      "SubRES"
+    ],
+    "valid_fields": [
+      {
+        "name": "commname",
+        "aliases": [
+          "commodity"
+        ],
+        "use_name": "other_indexes",
+        "row_ignore_symbol": [
+          "\\I:",
+          "*"
+        ],
+        "query_field": false,
+        "inherit_above": false,
+        "remove_first_row_if_absent": false,
+        "remove_any_row_if_absent": true
+      }
+    ]
+  },
   {
     "tag_name": "comemi",
     "tag_allowed_in": [
@@ -53,7 +77,7 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": true
       }
     ]
   },
diff --git a/xl2times/transforms.py b/xl2times/transforms.py
index 22eaa896..3f09a564 100644
--- a/xl2times/transforms.py
+++ b/xl2times/transforms.py
@@ -1624,31 +1624,44 @@ def remove_fill_tables(
     return result
 
 
-def process_commodity_emissions(
+def convert_com_tables(
     config: Config,
     tables: list[EmbeddedXlTable],
     model: TimesModel,
 ) -> list[EmbeddedXlTable]:
-    """Transform comemi tables to fi_t."""
+    """Transform comemi and comagg tables to fi_t."""
+    convert_tags = {
+        Tag.comemi: {
+            "attribute": "vda_emcb",
+            "index_column": "commodity",
+            "other_column": "other_indexes",
+        },
+        Tag.comagg: {
+            "attribute": "com_agg",
+            "index_column": "other_indexes",
+            "other_column": "commodity",
+        },
+    }
     result = []
     for table in tables:
-        if table.tag != Tag.comemi:
+        if table.tag not in convert_tags:
             result.append(table)
         else:
+            info = convert_tags[table.tag]
+            index_column = info["index_column"]
+            other_column = info["other_column"]
             df = table.dataframe.copy()
             # Remove columns that are not allowed
             # TODO: Base this on the config file instead
             remove_cols = ["region", "year"]
             df.drop(columns=remove_cols, errors="ignore", inplace=True)
-
-            index_columns = ["commodity"]
             data_columns = [
-                colname for colname in df.columns if colname not in index_columns
+                colname for colname in df.columns if colname != index_column
             ]
             df, names = utils.explode(df, data_columns)
-            df.rename(columns={"value": "vda_emcb"}, inplace=True)
-            df["other_indexes"] = names
-            df["other_indexes"] = df["other_indexes"].str.upper()
+            df.rename(columns={"value": info["attribute"]}, inplace=True)
+            df[other_column] = names
+            df[other_column] = df[other_column].str.upper()
 
             df = df.reset_index(drop=True)
             result.append(replace(table, dataframe=df, tag=Tag.fi_t))