Capitalise all string values in tables

utils/run_benchmarks.py

@@ -118,6 +118,7 @@ def run_gams_gdxdiff(
             path.join(out_folder, "diffile.gdx"),
             "Eps=0.000001",
             "RelEps=0.000001",
+            "setDesc=N",
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,

xl2times/__main__.py

@@ -115,6 +115,7 @@ def convert_xl_to_times(
         transforms.normalize_column_aliases,
         transforms.remove_comment_rows,
         transforms.revalidate_input_tables,
+        transforms.capitalise_table_values,
         transforms.process_regions,
         transforms.process_commodities,
         transforms.process_time_periods,
@@ -135,7 +136,6 @@ def convert_xl_to_times(
         transforms.generate_uc_properties,
         transforms.expand_rows_parallel,  # slow
         transforms.remove_invalid_values,
-        transforms.capitalise_some_values,
         transforms.internalise_commodities,
         transforms.generate_commodity_groups,
         transforms.apply_fixups,
@@ -422,7 +422,7 @@ def strip_filename_prefix(table, prefix):
 
 def dump_tables(tables: list, filename: str) -> list:
     os.makedirs(os.path.dirname(filename), exist_ok=True)
-    with open(filename, "w") as text_file:
+    with open(filename, "w", encoding="utf-8") as text_file:
         for t in tables if isinstance(tables, list) else tables.items():
             if isinstance(t, EmbeddedXlTable):
                 tag = t.tag

xl2times/transforms.py

@@ -1186,28 +1186,30 @@ def complete_dictionary(
     return tables
 
 
-def capitalise_some_values(
+def capitalise_table_values(
     config: Config,
     tables: list[EmbeddedXlTable],
     model: TimesModel,
 ) -> list[EmbeddedXlTable]:
-    """Ensure that all attributes and units are uppercase."""
-    # TODO: This should include other dimensions
-    # TODO: This should be part of normalisation
+    """Ensure that all table entries are uppercase. Strip leading and trailing whitespace."""
 
-    colnames = ["attribute", "tact", "tcap", "unit", "sourcescen"]
-
-    def capitalise_attributes_table(table: EmbeddedXlTable):
+    def capitalise_table_entries(table: EmbeddedXlTable):
         df = table.dataframe.copy()
+        # Capitalise all entries if column type string
+        colnames = df.select_dtypes(include="object").columns
         seen_cols = [colname for colname in colnames if colname in df.columns]
         if len(df) > 0:
             for seen_col in seen_cols:
-                df[seen_col] = df[seen_col].str.upper()
+                # Index of rows with string entries
+                i = df[seen_col].apply(lambda x: isinstance(x, str))
+                if any(i):
+                    df.loc[i, seen_col] = df[seen_col][i].str.upper()
+                    df.loc[i, seen_col] = df[seen_col][i].str.strip()
             return replace(table, dataframe=df)
         else:
             return table
 
-    return [capitalise_attributes_table(table) for table in tables]
+    return [capitalise_table_entries(table) for table in tables]
 
 
 def _populate_defaults(
@@ -1804,7 +1806,7 @@ def generate_dummy_processes(
         # TODO: Activity units below are arbitrary. Suggest Veda devs not to have any.
         dummy_processes = [
             ["IMP", "IMPNRGZ", "Dummy Import of NRG", "PJ", "", "NRG"],
-            ["IMP", "IMPMATZ", "Dummy Import of MAT", "Mt", "", "MAT"],
+            ["IMP", "IMPMATZ", "Dummy Import of MAT", "MT", "", "MAT"],
             ["IMP", "IMPDEMZ", "Dummy Import of DEM", "PJ", "", "DEM"],
         ]