Merge branch 'main' into olex/vda_flop

tests/test_transforms.py

@@ -9,7 +9,7 @@
     expand_rows,
     get_matching_commodities,
     get_matching_processes,
-    _match_uc_wildcards,
+    _match_wildcards,
     process_map,
     commodity_map,
 )
@@ -87,10 +87,10 @@ def test_uc_wildcards(self):
         t0 = datetime.now()
 
         # optimised functions
-        df_new = _match_uc_wildcards(
+        df_new = _match_wildcards(
             df, process_map, dictionary, get_matching_processes, "process"
         )
-        df_new = _match_uc_wildcards(
+        df_new = _match_wildcards(
             df_new, commodity_map, dictionary, get_matching_commodities, "commodity"
         )
 

xl2times/__main__.py

@@ -97,7 +97,7 @@ def convert_xl_to_times(
         transforms.remove_exreg_cols,
         transforms.generate_dummy_processes,
         transforms.process_time_slices,
-        transforms.process_transform_insert_variants,
+        transforms.process_transform_table_variants,
         transforms.process_transform_tables,
         transforms.process_tradelinks,
         transforms.process_processes,
@@ -119,12 +119,11 @@ def convert_xl_to_times(
         transforms.include_tables_source,
         transforms.merge_tables,
         transforms.complete_processes,
-        transforms.apply_more_fixups,
         transforms.process_units,
-        transforms.process_years,
         transforms.complete_commodity_groups,
-        transforms.process_uc_wildcards,
         transforms.process_wildcards,
+        transforms.apply_transform_tables,
+        transforms.apply_final_fixup,
         transforms.convert_aliases,
         transforms.fix_topology,
         transforms.resolve_remaining_cgs,
@@ -332,6 +331,7 @@ def produce_times_tables(
 def write_dd_files(
     tables: Dict[str, DataFrame], config: datatypes.Config, output_dir: str
 ):
+    encoding = "utf-8"
     os.makedirs(output_dir, exist_ok=True)
     for item in os.listdir(output_dir):
         if item.endswith(".dd"):
@@ -369,11 +369,13 @@ def convert_parameter(tablename: str, df: DataFrame):
     tables_in_file = {
         "ts.dd": ["ALL_TS"],
         "milestonyr.dd": ["MILESTONYR"],
-        "output.dd": [t for t in config.dd_table_order if t != "ALL_TS"],
+        "output.dd": [
+            t for t in config.dd_table_order if t not in ["ALL_TS", "MILESTONYR"]
+        ],
     }
 
     for fname, tablenames in tables_in_file.items():
-        with open(os.path.join(output_dir, fname), "w") as fout:
+        with open(os.path.join(output_dir, fname), "w", encoding=encoding) as fout:
             for tablename in [t for t in tablenames if t in tables]:
                 df = tables[tablename]
                 if tablename in sets:

xl2times/config/veda-tags.json

@@ -1008,7 +1008,7 @@
           "cset:cn",
           "cset: cn"
         ],
-        "use_name": "cset_cn",
+        "use_name": "commodity",
         "row_ignore_symbol": [
           "\\I:"
         ],
@@ -1072,7 +1072,7 @@
           "pset:pn",
           "pset: pn"
         ],
-        "use_name": "pset_pn",
+        "use_name": "process",
         "row_ignore_symbol": [
           "\\I:"
         ],

xl2times/datatypes.py

@@ -6,7 +6,6 @@
 import re
 from typing import Dict, Iterable, List, Set, Tuple
 from enum import Enum
-
 from loguru import logger
 from pandas.core.frame import DataFrame
 
@@ -161,14 +160,38 @@ class TimesModel:
     time_periods: DataFrame = field(default_factory=DataFrame)
     units: DataFrame = field(default_factory=DataFrame)
     start_year: int = field(default_factory=int)
-    data_years: Tuple[int] = field(default_factory=tuple)
-    model_years: Tuple[int] = field(default_factory=tuple)
-    past_years: Tuple[int] = field(default_factory=tuple)
     files: Set[str] = field(default_factory=set)
 
+    @property
     def external_regions(self) -> Set[str]:
         return self.all_regions.difference(self.internal_regions)
 
+    @property
+    def data_years(self) -> Set[int]:
+        """
+        data_years are years for which there is data specified.
+        """
+        data_years = set()
+        for attributes in [self.attributes, self.uc_attributes]:
+            if not attributes.empty:
+                data_years.update(attributes["year"].astype(int).values)
+        # Remove interpolation rules before return
+        return {y for y in data_years if y >= 1000}
+
+    @property
+    def past_years(self) -> Set[int]:
+        """
+        Pastyears is the set of all years before start_year.
+        """
+        return {x for x in self.data_years if x < self.start_year}
+
+    @property
+    def model_years(self) -> Set[int]:
+        """
+        model_years is the union of past_years and the representative years of the model (middleyears).
+        """
+        return self.past_years | set(self.time_periods["m"].values)
+
 
 class Config:
     """Encapsulates all configuration options for a run of the tool, including