Use times-info file for parameter mappings (#148)

--------- Co-authored-by: Olexandr Balyk <[email protected]>
etsap-TIMES · Dec 15, 2023 · 5a6e2cb · 5a6e2cb
1 parent dcc67e9
commit 5a6e2cb
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 75 deletions.
diff --git a/xl2times/__main__.py b/xl2times/__main__.py
@@ -193,30 +193,30 @@ def compare(
                     f"WARNING: Table {table_name} header incorrect, was"
                     f" {data_cols}, should be {transformed_gt_cols}"
                 )
-            else:
-                # both are in string form so can be compared without any issues
-                gt_rows = set(tuple(row) for row in gt_table.to_numpy().tolist())
-                data_rows = set(tuple(row) for row in data_table.to_numpy().tolist())
-                total_correct_rows += len(gt_rows.intersection(data_rows))
-                additional = data_rows - gt_rows
-                total_additional_rows += len(additional)
-                missing = gt_rows - data_rows
-                if len(additional) != 0 or len(missing) != 0:
-                    print(
-                        f"WARNING: Table {table_name} ({data_table.shape[0]} rows,"
-                        f" {gt_table.shape[0]} GT rows) contains {len(additional)}"
-                        f" additional rows and is missing {len(missing)} rows"
-                    )
-                if len(additional) != 0:
-                    DataFrame(additional).to_csv(
-                        os.path.join(output_dir, table_name + "_additional.csv"),
-                        index=False,
-                    )
-                if len(missing) != 0:
-                    DataFrame(missing).to_csv(
-                        os.path.join(output_dir, table_name + "_missing.csv"),
-                        index=False,
-                    )
+
+            # both are in string form so can be compared without any issues
+            gt_rows = set(tuple(row) for row in gt_table.to_numpy().tolist())
+            data_rows = set(tuple(row) for row in data_table.to_numpy().tolist())
+            total_correct_rows += len(gt_rows.intersection(data_rows))
+            additional = data_rows - gt_rows
+            total_additional_rows += len(additional)
+            missing = gt_rows - data_rows
+            if len(additional) != 0 or len(missing) != 0:
+                print(
+                    f"WARNING: Table {table_name} ({data_table.shape[0]} rows,"
+                    f" {gt_table.shape[0]} GT rows) contains {len(additional)}"
+                    f" additional rows and is missing {len(missing)} rows"
+                )
+            if len(additional) != 0:
+                DataFrame(additional).to_csv(
+                    os.path.join(output_dir, table_name + "_additional.csv"),
+                    index=False,
+                )
+            if len(missing) != 0:
+                DataFrame(missing).to_csv(
+                    os.path.join(output_dir, table_name + "_missing.csv"),
+                    index=False,
+                )
 
     print(
         f"{total_correct_rows / total_gt_rows :.1%} of ground truth rows present"

diff --git a/xl2times/config/times-info.json b/xl2times/config/times-info.json
@@ -137,7 +137,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -155,7 +155,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -298,6 +298,7 @@
   {
     "name": "B",
     "gams-cat": "parameter",
+    "type": "derived",
     "indexes": [
       "YEAR"
     ],
@@ -696,7 +697,7 @@
     "mapping": [
       "region",
       "commodity",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -764,7 +765,7 @@
       "COM_GRP"
     ],
     "mapping": [
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -1031,7 +1032,7 @@
     ],
     "mapping": [
       "region",
-      "commodity_group",
+      "other_indexes",
       "commodity"
     ]
   },
@@ -1098,7 +1099,7 @@
     ],
     "mapping": [
       "region",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -1141,7 +1142,7 @@
     ],
     "mapping": [
       "region",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -1428,6 +1429,7 @@
   {
     "name": "E",
     "gams-cat": "parameter",
+    "type": "derived",
     "indexes": [
       "YEAR"
     ],
@@ -1468,7 +1470,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice",
       "limtype"
     ]
@@ -1548,7 +1550,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "commodity",
       "timeslice"
     ]
@@ -1568,7 +1570,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "commodity",
       "timeslice"
     ]
@@ -1624,8 +1626,8 @@
       "region",
       "year",
       "process",
-      "commodity_group",
-      "commodity_group",
+      "other_indexes",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -1643,8 +1645,8 @@
       "region",
       "year",
       "process",
-      "commodity_group",
-      "commodity_group"
+      "other_indexes",
+      "other_indexes"
     ]
   },
   {
@@ -1700,7 +1702,7 @@
       "year",
       "process",
       "commodity",
-      "commodity_group",
+      "other_indexes",
       "timeslice",
       "limtype"
     ]
@@ -1741,9 +1743,9 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "commodity",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -1772,11 +1774,11 @@
     "gams-cat": "parameter",
     "indexes": [
       "CUR",
-      "CUR"
+      "CUR2"
     ],
     "mapping": [
       "currency",
-      "currency"
+      "other_indexes"
     ]
   },
   {
@@ -1900,7 +1902,7 @@
       "region",
       "year",
       "commodity",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -2206,7 +2208,7 @@
       "region",
       "year",
       "process",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -2223,7 +2225,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -2241,7 +2243,7 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   },
@@ -2506,7 +2508,7 @@
       "region",
       "year",
       "process",
-      "tbd"
+      "other_indexes"
     ]
   },
   {
@@ -3052,7 +3054,7 @@
       "region",
       "year",
       "process",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -3067,7 +3069,7 @@
     "mapping": [
       "region",
       "process",
-      "commodity_group",
+      "other_indexes",
       "tbd"
     ]
   },
@@ -3111,7 +3113,7 @@
     "mapping": [
       "region",
       "process",
-      "commodity_group",
+      "other_indexes",
       "tbd"
     ]
   },
@@ -3747,8 +3749,8 @@
       "region",
       "year",
       "process",
-      "commodity_group",
-      "commodity_group",
+      "other_indexes",
+      "other_indexes",
       "stage",
       "sow"
     ]
@@ -3925,7 +3927,7 @@
       "AGE"
     ],
     "mapping": [
-      "stage",
+      "other_indexes",
       "year"
     ]
   },
@@ -4401,7 +4403,7 @@
     ],
     "mapping": [
       "region",
-      "commodity_group",
+      "other_indexes",
       "limtype"
     ]
   },
@@ -4417,7 +4419,7 @@
     "mapping": [
       "region",
       "year",
-      "commodity_group",
+      "other_indexes",
       "limtype"
     ]
   },
@@ -4752,7 +4754,7 @@
       "region",
       "year",
       "process",
-      "commodity_group"
+      "other_indexes"
     ]
   },
   {
@@ -5102,6 +5104,7 @@
   {
     "name": "VDA_EMCB",
     "gams-cat": "parameter",
+    "type": "special",
     "indexes": [
       "REG",
       "YEAR",
@@ -5111,7 +5114,7 @@
     "mapping": [
       "region",
       "year",
-      "commodity",
+      "other_indexes",
       "commodity"
     ]
   },
@@ -5129,8 +5132,8 @@
       "region",
       "year",
       "process",
-      "commodity_group",
+      "other_indexes",
       "timeslice"
     ]
   }
-]
+]