From c3d8eece7f21698373b032dd4fe06225fa5b7ab9 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Fri, 1 Dec 2023 16:40:48 +0000 Subject: [PATCH 1/5] Normalizing columns: add some checks (#138) This is some stuff I forgot to include in #135 --- times_reader/datatypes.py | 19 ++++++++++++++++--- times_reader/transforms.py | 4 ++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/times_reader/datatypes.py b/times_reader/datatypes.py index b893ca5..27449b8 100644 --- a/times_reader/datatypes.py +++ b/times_reader/datatypes.py @@ -61,7 +61,7 @@ class Tag(str, Enum): uc_sets = "~UC_SETS" uc_t = "~UC_T" # This is used by Veda for unit conversion when displaying results - # unitconversion = "~UNITCONVERSION" + unitconversion = "~UNITCONVERSION" @classmethod def has_tag(cls, tag): @@ -270,15 +270,28 @@ def _read_mappings(filename: str) -> List[TimesXlMap]: def _read_veda_tags_info( veda_tags_file: str, ) -> Tuple[Dict[Tag, Dict[str, str]], Dict[Tag, Dict[str, list]]]: + def to_tag(s: str) -> Tag: + # The file stores the tag name in lowercase, and without the ~ + return Tag("~" + s.upper()) + + # Read veda_tags_file with resources.open_text("times_reader.config", veda_tags_file) as f: veda_tags_info = json.load(f) + + # Check that all the tags we use are present in veda_tags_file + tags = {to_tag(tag_info["tag_name"]) for tag_info in veda_tags_info} + for tag in Tag: + if tag not in tags: + print( + f"WARNING: datatypes.Tag has an unknown Tag {tag} not in {veda_tags_file}" + ) + column_aliases = {} row_comment_chars = {} for tag_info in veda_tags_info: if "tag_fields" in tag_info: - # The file stores the tag name in lowercase, and without the ~ - tag_name = "~" + tag_info["tag_name"].upper() + tag_name = to_tag(tag_info["tag_name"]) # Process column aliases: column_aliases[tag_name] = {} names = tag_info["tag_fields"]["fields_names"] diff --git a/times_reader/transforms.py b/times_reader/transforms.py index 57d4bd6..448eabf 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -229,6 +229,10 @@ def normalize_column_aliases( ) else: print(f"WARNING: could not find {table.tag} in config.column_aliases") + if len(set(table.dataframe.columns)) > len(table.dataframe.columns): + raise ValueError( + f"Table has duplicate column names (after normalization): {table}" + ) return tables From 06cd28f327f1d76bcf92c3698dd4db0c843654d2 Mon Sep 17 00:00:00 2001 From: Siddharth Krishna Date: Fri, 1 Dec 2023 16:41:23 +0000 Subject: [PATCH 2/5] Check for dupes and drop empties in new validate transform (#145) --- times_reader/__main__.py | 2 +- times_reader/transforms.py | 38 +++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/times_reader/__main__.py b/times_reader/__main__.py index 43dd924..9ffd7c4 100644 --- a/times_reader/__main__.py +++ b/times_reader/__main__.py @@ -56,7 +56,7 @@ def convert_xl_to_times( transform_list = [ transforms.normalize_tags_columns, transforms.remove_fill_tables, - transforms.remove_empty_tables, + transforms.validate_input_tables, lambda config, tables: [transforms.remove_comment_cols(t) for t in tables], transforms.remove_tables_with_formulas, # slow transforms.normalize_column_aliases, diff --git a/times_reader/transforms.py b/times_reader/transforms.py index 448eabf..eb6c171 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -102,15 +102,6 @@ def remove_comment_cols(table: datatypes.EmbeddedXlTable) -> datatypes.EmbeddedX df = table.dataframe.drop(comment_cols, axis=1) df.reset_index(drop=True, inplace=True) - - # TODO: should we move the code below to a separate transform? - seen = set() - dupes = [x for x in df.columns if x in seen or seen.add(x)] - if len(dupes) > 0: - print( - f"WARNING: Duplicate columns in {table.range}, {table.sheetname}," - f" {table.filename}: {','.join(dupes)}" - ) return replace(table, dataframe=df) @@ -138,15 +129,13 @@ def has_formulas(table): return [table for table in tables if not has_formulas(table)] -def remove_empty_tables( +def validate_input_tables( config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable], ) -> List[datatypes.EmbeddedXlTable]: """ - Return a modified copy of 'tables' where empty tables have been deleted from the list. - - :param tables: List of tables in EmbeddedXlTable format. - :return: List of non-empty tables in EmbeddedXlTable format. + Perform some basic validation (tag names are valid, no duplicate column labels), and + remove empty tables (for recognized tags). """ check_list = [ @@ -181,10 +170,29 @@ def remove_empty_tables( def discard(table): if table.tag in check_list: return not table.dataframe.shape[0] + elif table.tag == datatypes.Tag.unitconversion: + print("Dropping ~UNITCONVERSION table") + return True else: return False - return [table for table in tables if not discard(table)] + result = [] + for table in tables: + if not datatypes.Tag.has_tag(table.tag.split(":")[0]): + print(f"WARNING: Dropping table with unrecognized tag {table.tag}") + continue + if discard(table): + continue + # Check for duplicate columns: + seen = set() + dupes = [x for x in table.dataframe.columns if x in seen or seen.add(x)] + if len(dupes) > 0: + print( + f"WARNING: Duplicate columns in {table.range}, {table.sheetname}," + f" {table.filename}: {','.join(dupes)}" + ) + result.append(table) + return result def normalize_tags_columns( From 073dd9fcf8fe3dafa7b9157c3f415d8d55143c2a Mon Sep 17 00:00:00 2001 From: Olexandr Balyk Date: Fri, 1 Dec 2023 22:08:45 -0500 Subject: [PATCH 3/5] Account for aliases when processing AT tables (#144) --- times_reader/datatypes.py | 11 ++++++++--- times_reader/transforms.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/times_reader/datatypes.py b/times_reader/datatypes.py index 27449b8..5d23f33 100644 --- a/times_reader/datatypes.py +++ b/times_reader/datatypes.py @@ -146,6 +146,7 @@ class Config: times_xl_maps: List[TimesXlMap] dd_table_order: Iterable[str] all_attributes: Set[str] + attr_aliases: Set[str] # For each tag, this dictionary maps each column alias to the normalized name column_aliases: Dict[Tag, Dict[str, str]] # For each tag, this dictionary specifies comment row symbols by column name @@ -166,7 +167,7 @@ def __init__( self.column_aliases, self.row_comment_chars = Config._read_veda_tags_info( veda_tags_file ) - self.veda_attr_defaults = Config._read_veda_attr_defaults( + self.veda_attr_defaults, self.attr_aliases = Config._read_veda_attr_defaults( veda_attr_defaults_file ) @@ -311,7 +312,7 @@ def to_tag(s: str) -> Tag: @staticmethod def _read_veda_attr_defaults( veda_attr_defaults_file: str, - ) -> Dict[str, Dict[str, list]]: + ) -> Tuple[Dict[str, Dict[str, list]], Set[str]]: # Read veda_tags_file with resources.open_text("times_reader.config", veda_attr_defaults_file) as f: defaults = json.load(f) @@ -323,6 +324,10 @@ def _read_veda_attr_defaults( "tslvl": {"DAYNITE": [], "ANNUAL": []}, } + attr_aliases = { + attr for attr in defaults if "times-attribute" in defaults[attr] + } + for attr, attr_info in defaults.items(): # Populate aliases by attribute dictionary if "times-attribute" in attr_info: @@ -343,4 +348,4 @@ def _read_veda_attr_defaults( tslvl = attr_defaults["ts-level"] veda_attr_defaults["tslvl"][tslvl].append(attr) - return veda_attr_defaults + return veda_attr_defaults, attr_aliases diff --git a/times_reader/transforms.py b/times_reader/transforms.py index eb6c171..669f669 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -1605,7 +1605,7 @@ def is_year(col_name): other_columns = [ col_name for col_name in df.columns - if col_name not in config.all_attributes + if col_name not in (config.all_attributes | config.attr_aliases) ] df = pd.melt( df, From 4dade4b2c31314ff9ea5ceb65c60a81cbe02b55c Mon Sep 17 00:00:00 2001 From: Olexandr Balyk Date: Sat, 2 Dec 2023 01:45:03 -0500 Subject: [PATCH 4/5] Don't drop duplicates in fi_process table when generating topology (#146) --- times_reader/__main__.py | 2 +- times_reader/transforms.py | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/times_reader/__main__.py b/times_reader/__main__.py index 9ffd7c4..acb1f72 100644 --- a/times_reader/__main__.py +++ b/times_reader/__main__.py @@ -80,7 +80,7 @@ def convert_xl_to_times( transforms.remove_invalid_values, transforms.process_time_periods, transforms.generate_all_regions, - transforms.capitalise_attributes, + transforms.capitalise_some_values, transforms.apply_fixups, transforms.generate_commodity_groups, transforms.fill_in_missing_pcgs, diff --git a/times_reader/transforms.py b/times_reader/transforms.py index 669f669..66dc155 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -742,6 +742,7 @@ def fill_in_missing_values_table(table): df[colname].fillna(start_year, inplace=True) elif colname == "currency": df[colname].fillna(currency, inplace=True) + return replace(table, dataframe=df) for table in tables: @@ -907,20 +908,25 @@ def generate_all_regions( return tables -def capitalise_attributes( +def capitalise_some_values( config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable], ) -> List[datatypes.EmbeddedXlTable]: """ - Ensure that all attributes are uppercase + Ensure that all attributes and units are uppercase """ # TODO: This should include other dimensions # TODO: This should be part of normalisation + + colnames = ["attribute", "tact", "tcap", "unit"] + def capitalise_attributes_table(table: datatypes.EmbeddedXlTable): df = table.dataframe.copy() - if "attribute" in df.columns and len(df) > 0: - df["attribute"] = df["attribute"].str.upper() + seen_cols = [colname for colname in colnames if colname in df.columns] + if len(df) > 0: + for seen_col in seen_cols: + df[seen_col] = df[seen_col].str.upper() return replace(table, dataframe=df) else: return table @@ -1818,17 +1824,10 @@ def generate_topology_dictionary(tables: Dict[str, DataFrame]) -> Dict[str, Data processes = tables[datatypes.Tag.fi_process] commodities = tables[datatypes.Tag.fi_comm] - duplicated_processes = processes[["process"]].duplicated() - if any(duplicated_processes): - duplicated_process_names = processes["process"][duplicated_processes] - print( - f"WARNING: {len(duplicated_process_names)} duplicated processes: {duplicated_process_names.values[1:3]}" - ) - processes.drop_duplicates(subset="process", inplace=True) - dictionary["processes_by_name"] = ( processes[["process"]] .dropna() + .drop_duplicates() .set_index("process", drop=False) .rename_axis("index") ) From 8869a687006b477527d4b7537e11ead46dd710bf Mon Sep 17 00:00:00 2001 From: Olexandr Balyk Date: Tue, 5 Dec 2023 20:50:11 -0500 Subject: [PATCH 5/5] Simplify normalisation (#147) This PR: - restructures veda-tags.json - removes apply_postnormalisation_fixes - moves info on which tables can be discarded if empty to `config` --------- Co-authored-by: Siddharth Krishna --- times_reader/__main__.py | 1 - times_reader/config/veda-tags.json | 3751 +++++++++++++++++----------- times_reader/datatypes.py | 64 +- times_reader/transforms.py | 65 +- 4 files changed, 2311 insertions(+), 1570 deletions(-) diff --git a/times_reader/__main__.py b/times_reader/__main__.py index acb1f72..ffba993 100644 --- a/times_reader/__main__.py +++ b/times_reader/__main__.py @@ -63,7 +63,6 @@ def convert_xl_to_times( lambda config, tables: [ transforms.remove_comment_rows(config, t) for t in tables ], - transforms.apply_postnormalisation_fixes, transforms.generate_dummy_processes, transforms.process_transform_insert_variants, transforms.process_transform_insert, diff --git a/times_reader/config/veda-tags.json b/times_reader/config/veda-tags.json index 0a2fe5a..70b51a4 100644 --- a/times_reader/config/veda-tags.json +++ b/times_reader/config/veda-tags.json @@ -10,20 +10,20 @@ "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "bookname", - "region" - ], - "fields_aliases": [ - [], - [] - ], - "row_ignore_symbol": [ - [], - [] - ] - } + "valid_fields": [ + { + "name": "bookname", + "aliases": [], + "use_name": "bookname", + "row_ignore_symbol": [] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [] + } + ] }, { "tag_name": "comemi", @@ -31,22 +31,19 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "commname" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "commname", + "aliases": [ "commodity" - ] - ], - "row_ignore_symbol": [ - [ + ], + "use_name": "commodity", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "currencies", @@ -79,80 +76,101 @@ "SubRES", "SysSettings" ], - "tag_fields": { - "fields_names": [ - "commdesc", - "commname", - "csets", - "ctslvl", - "ctype", - "limtype", - "peakts", - "region", - "unit" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "commdesc", + "aliases": [ "description" ], - [ + "use_name": "commdesc", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "commname", + "aliases": [ "commodity" ], - [ + "use_name": "commodity", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "csets", + "aliases": [ "cset", "sets", "set" ], - [ + "use_name": "csets", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "ctslvl", + "aliases": [ "tslvl", "timeslicelevel" ], - [ + "use_name": "ctslvl", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "ctype", + "aliases": [ "type", "commoditytype" ], - [ + "use_name": "ctype", + "row_ignore_symbol": [] + }, + { + "name": "limtype", + "aliases": [ "bd" ], - [ - "peaktimeslice" - ], - [], - [] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [ + "use_name": "limtype", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "peakts", + "aliases": [ + "peaktimeslice" ], - [], - [ + "use_name": "peakts", + "row_ignore_symbol": [] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "unit", + "aliases": [], + "use_name": "unit", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "fi_process", @@ -160,83 +178,104 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "primarycg", - "region", - "sets", - "tact", - "tcap", - "techdesc", - "techname", - "tslvl", - "vintage" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "primarycg", + "aliases": [ "pcg", "primarycommoditygroup", "primary_commodity_group" ], - [], - [ - "set" - ], - [ - "activityunit", - "activity_unit" - ], - [ - "capacityunit", - "capacity_unit" - ], - [ - "description" - ], - [ - "process" - ], - [ - "timeslicelevel" - ], - [] - ], - "row_ignore_symbol": [ - [ + "use_name": "primarycg", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "sets", + "aliases": [ + "set" ], - [ + "use_name": "sets", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tact", + "aliases": [ + "activityunit", + "activity_unit" ], - [ + "use_name": "tact", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tcap", + "aliases": [ + "capacityunit", + "capacity_unit" ], - [ + "use_name": "tcap", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "techdesc", + "aliases": [ + "description" ], - [ + "use_name": "techdesc", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "techname", + "aliases": [ + "process" ], - [ + "use_name": "process", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tslvl", + "aliases": [ + "timeslicelevel" ], - [ + "use_name": "tslvl", + "row_ignore_symbol": [ "\\I:", "*" - ], - [] - ] - } + ] + }, + { + "name": "vintage", + "aliases": [], + "use_name": "vintage", + "row_ignore_symbol": [] + } + ] }, { "tag_name": "fi_t", @@ -244,43 +283,49 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "attribute", - "commodity", - "commodity-in", - "commodity-in-aux", - "commodity-out", - "commodity-out-aux", - "currency", - "limtype", - "other_indexes", - "process", - "region", - "sow", - "stage", - "timeslice", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attribute", + "aliases": [ "parameter", "prmtr", "attr", "attrib" ], - [ + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "commodity", + "aliases": [ "commname", "cset_cn" ], - [ + "use_name": "commodity", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-in", + "aliases": [ "comm-in", "comm_in", "commin", "commodity_in", "commodityin" ], - [ + "use_name": "commodity-in", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-in-aux", + "aliases": [ "comm-in-a", "comm_in-a", "commin-a", @@ -288,14 +333,28 @@ "commodityin-aux", "commodity_in_aux" ], - [ + "use_name": "commodity-in-aux", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-out", + "aliases": [ "comm-out", "comm_out", "commout", "commodity_out", "commodityout" ], - [ + "use_name": "commodity-out", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-out-aux", + "aliases": [ "comm-out-a", "comm_out-a", "commout-a", @@ -303,93 +362,112 @@ "commodityout-aux", "commodity_out_aux" ], - [ + "use_name": "commodity-out-aux", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "currency", + "aliases": [ "curr" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind", "commgrp", "commodity_group" ], - [ - "techname", - "pset_pn" - ], - [], - [], - [], - [ - "timeslices", - "time_slice", - "time_slices", - "ts" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "process", + "aliases": [ + "techname", + "pset_pn" ], - [ + "use_name": "process", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice", + "aliases": [ + "timeslices", + "time_slice", + "time_slices", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "milestoneyears", @@ -408,200 +486,254 @@ "tag_allowed_in": [ "SR_Trans" ], - "tag_fields": { - "fields_names": [ - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_comgrp", "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "cset_cd", - "cset_cn", - "cset_set", - "description", - "name", - "value" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [], - [], - [ - "valfield", - "val_field", - "allregions" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "description", + "aliases": [], + "use_name": "description", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "name", + "aliases": [], + "use_name": "name", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_csets", "tag_allowed_in": [ "setrules" ], - "tag_fields": { - "fields_names": [ - "and_or_negative_dimension", - "and_or_negative_for_sets", - "and_or_positive_dimension", - "and_or_positive_for_sets", - "comment", - "description", - "dimension_name", - "dimension_set_name", - "input_commodity", - "name", - "outout_commodity", - "set_description", - "set_name" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "and_or_negative_dimension", + "aliases": [ "c_neg_andor" ], - [ + "use_name": "and_or_negative_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_negative_for_sets", + "aliases": [ "c_neg_andor_forsets" ], - [ + "use_name": "and_or_negative_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_dimension", + "aliases": [ "c_pos_andor" ], - [ + "use_name": "and_or_positive_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_for_sets", + "aliases": [ "c_pos_andor_forsets" ], - [ + "use_name": "and_or_positive_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "comment", + "aliases": [ "comments" ], - [ + "use_name": "comment", + "row_ignore_symbol": [] + }, + { + "name": "description", + "aliases": [ "cset_cd" ], - [], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [] + }, + { + "name": "dimension_name", + "aliases": [], + "use_name": "dimension_name", + "row_ignore_symbol": [] + }, + { + "name": "dimension_set_name", + "aliases": [ "cset_set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [] + }, + { + "name": "input_commodity", + "aliases": [ "pset_ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [] + }, + { + "name": "name", + "aliases": [ "cset_cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [] + }, + { + "name": "outout_commodity", + "aliases": [ "pset_co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [] + }, + { + "name": "set_description", + "aliases": [ "setdesc", "set_desc" ], - [ + "use_name": "set_description", + "row_ignore_symbol": [] + }, + { + "name": "set_name", + "aliases": [ "setname" + ], + "use_name": "set_name", + "row_ignore_symbol": [ + "\\I:", + "*" ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "\\I:", - "*" - ] - ] - } + } + ] }, { "tag_name": "tfm_dins", @@ -612,128 +744,167 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attribute", - "cset_cn", - "curr", - "limtype", - "other_indexes", - "pset_pn", - "region", - "side", - "sow", - "stage", - "timeslice", - "ts_filter", - "uc_n", - "value", - "year" - ], - "fields_aliases": [ - [], - [ + "valid_fields": [ + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [], - [ - "user_constraint" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "ts_filter", + "aliases": [], + "use_name": "ts_filter", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_dins-at", @@ -781,239 +952,335 @@ "ParScen", "SysSettings" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "operation_sum_avg_count", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "sourcescen", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "operation_sum_avg_count", + "aliases": [], + "use_name": "operation_sum_avg_count", + "row_ignore_symbol": [] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [ - "scenario", - "scenario name", - "scen" - ], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "sourcescen", + "aliases": [ + "scenario", + "scenario name", + "scen" ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_ins", @@ -1024,243 +1291,342 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "ts_filter", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [], - [], - [], - [], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [ + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "ts_filter", + "aliases": [], + "use_name": "ts_filter", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_ins-at", @@ -1307,397 +1673,556 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "attribute2", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "commodity2", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "currency2", - "limtype", - "limtype2", - "other_indexes", - "other_indexes2", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sourcescen", - "sow", - "sow2", - "stage", - "stage2", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "timeslice2", - "top_check", - "uc_n", - "val_cond", - "value", - "year", - "year2" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute2", + "aliases": [], + "use_name": "attribute2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "commodity2", + "aliases": [ "cset_cn2", "commname2", "comm_name2", "cset:cn2", "cset: cn2" ], - [ + "use_name": "commodity2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "currency2", + "aliases": [ "curr2" ], - [ + "use_name": "currency2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype2", + "aliases": [ "lim_type2", "bd2" ], - [ + "use_name": "limtype2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes2", + "aliases": [ "other_ind2", "oth_ind2" ], - [ + "use_name": "other_indexes2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "time_slice2", - "ts2" - ], - [ - "top_chk" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ], - [ - "yr2" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sourcescen", + "aliases": [], + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "sow2", + "aliases": [], + "use_name": "sow2", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage2", + "aliases": [], + "use_name": "stage2", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice2", + "aliases": [ + "time_slice2", + "ts2" ], - [], - [], - [], - [], - [ + "use_name": "timeslice2", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "year2", + "aliases": [ + "yr2" ], - [ + "use_name": "year2", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_psets", "tag_allowed_in": [ "setrules" ], - "tag_fields": { - "fields_names": [ - "and_or_negative_dimension", - "and_or_negative_for_sets", - "and_or_positive_dimension", - "and_or_positive_for_sets", - "comment", - "description", - "dimension_name", - "dimension_set_name", - "input_commodity", - "name", - "outout_commodity", - "set_description", - "set_name" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "and_or_negative_dimension", + "aliases": [ "t_neg_andor" ], - [ + "use_name": "and_or_negative_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_negative_for_sets", + "aliases": [ "t_neg_andor_forsets" ], - [ + "use_name": "and_or_negative_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_dimension", + "aliases": [ "t_pos_andor" ], - [ + "use_name": "and_or_positive_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_for_sets", + "aliases": [ "t_pos_andor_forsets" ], - [ + "use_name": "and_or_positive_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "comment", + "aliases": [ "comments" ], - [ + "use_name": "comment", + "row_ignore_symbol": [] + }, + { + "name": "description", + "aliases": [ "pset_pd" ], - [], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [] + }, + { + "name": "dimension_name", + "aliases": [], + "use_name": "dimension_name", + "row_ignore_symbol": [] + }, + { + "name": "dimension_set_name", + "aliases": [ "pset_set" ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [] + }, + { + "name": "input_commodity", + "aliases": [ "pset_ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [] + }, + { + "name": "name", + "aliases": [ "pset_pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [] + }, + { + "name": "outout_commodity", + "aliases": [ "pset_co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [] + }, + { + "name": "set_description", + "aliases": [ "setdesc", "set_desc" ], - [ + "use_name": "set_description", + "row_ignore_symbol": [] + }, + { + "name": "set_name", + "aliases": [ "setname" + ], + "use_name": "set_name", + "row_ignore_symbol": [ + "\\I:", + "*" ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "\\I:", - "*" - ] - ] - } + } + ] }, { "tag_name": "tfm_topdins", @@ -1708,52 +2233,58 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "cset_cn", - "pset_pn", - "region", - "value" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "value", + "aliases": [ "valfield", "val_field", "allregions", "io" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_topins", @@ -1764,123 +2295,171 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "value" - ], - "fields_aliases": [ - [], - [], - [], - [], - [ + "valid_fields": [ + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "value", + "aliases": [ "valfield", "val_field", "allregions", "io" - ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [], - [], - [], - [], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_upd", @@ -1891,243 +2470,342 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sourcescen", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sourcescen", + "aliases": [], + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_upd-at", @@ -2165,286 +2843,391 @@ "SysSettings", "RegScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "currency", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "uc_attr", - "uc_desc", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [ "prmtr", "parameter" ], - [ + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "currency", + "aliases": [ "curr" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "uc-attr", - "ucattr" - ], - [ - "description", - "uc_description" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_attr", + "aliases": [ + "uc-attr", + "ucattr" ], - [], - [ + "use_name": "uc_attr", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_desc", + "aliases": [ + "description", + "uc_description" ], - [ + "use_name": "uc_desc", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "unitconversion", "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "from_unit", - "multiplier", - "to_unit" - ], - "fields_aliases": [ - [], - [], - [] - ], - "row_ignore_symbol": [ - [ + "valid_fields": [ + { + "name": "from_unit", + "aliases": [], + "use_name": "from_unit", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "multiplier", + "aliases": [], + "use_name": "multiplier", + "row_ignore_symbol": [] + }, + { + "name": "to_unit", + "aliases": [], + "use_name": "to_unit", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] } ] diff --git a/times_reader/datatypes.py b/times_reader/datatypes.py index 5d23f33..d5659db 100644 --- a/times_reader/datatypes.py +++ b/times_reader/datatypes.py @@ -151,6 +151,8 @@ class Config: column_aliases: Dict[Tag, Dict[str, str]] # For each tag, this dictionary specifies comment row symbols by column name row_comment_chars: Dict[Tag, Dict[str, list]] + # List of tags for which empty tables should be discarded + discard_if_empty: Iterable[Tag] veda_attr_defaults: Dict[str, Dict[str, list]] def __init__( @@ -164,9 +166,11 @@ def __init__( self.dd_table_order, self.all_attributes = Config._process_times_info( times_info_file ) - self.column_aliases, self.row_comment_chars = Config._read_veda_tags_info( - veda_tags_file - ) + ( + self.column_aliases, + self.row_comment_chars, + self.discard_if_empty, + ) = Config._read_veda_tags_info(veda_tags_file) self.veda_attr_defaults, self.attr_aliases = Config._read_veda_attr_defaults( veda_attr_defaults_file ) @@ -270,7 +274,7 @@ def _read_mappings(filename: str) -> List[TimesXlMap]: @staticmethod def _read_veda_tags_info( veda_tags_file: str, - ) -> Tuple[Dict[Tag, Dict[str, str]], Dict[Tag, Dict[str, list]]]: + ) -> Tuple[Dict[Tag, Dict[str, str]], Dict[Tag, Dict[str, list]], Iterable[Tag]]: def to_tag(s: str) -> Tag: # The file stores the tag name in lowercase, and without the ~ return Tag("~" + s.upper()) @@ -287,27 +291,45 @@ def to_tag(s: str) -> Tag: f"WARNING: datatypes.Tag has an unknown Tag {tag} not in {veda_tags_file}" ) - column_aliases = {} + valid_column_names = {} row_comment_chars = {} + discard_if_empty = [] for tag_info in veda_tags_info: - if "tag_fields" in tag_info: - tag_name = to_tag(tag_info["tag_name"]) - # Process column aliases: - column_aliases[tag_name] = {} - names = tag_info["tag_fields"]["fields_names"] - aliases = tag_info["tag_fields"]["fields_aliases"] - assert len(names) == len(aliases) - for name, aliases in zip(names, aliases): - for alias in aliases: - column_aliases[tag_name][alias] = name - # Process comment chars: + tag_name = to_tag(tag_info["tag_name"]) + if "valid_fields" in tag_info: + discard_if_empty.append(tag_name) + + valid_column_names[tag_name] = {} row_comment_chars[tag_name] = {} - chars = tag_info["tag_fields"]["row_ignore_symbol"] - assert len(names) == len(chars) - for name, chars_list in zip(names, chars): - row_comment_chars[tag_name][name] = chars_list - return column_aliases, row_comment_chars + # Process column aliases and comment chars: + for valid_field in tag_info["valid_fields"]: + valid_field_names = valid_field["aliases"] + if ( + "use_name" in valid_field + and valid_field["use_name"] != valid_field["name"] + ): + field_name = valid_field["use_name"] + valid_field_names.append(valid_field["name"]) + else: + field_name = valid_field["name"] + + for valid_field_name in valid_field_names: + valid_column_names[tag_name][valid_field_name] = field_name + row_comment_chars[tag_name][field_name] = valid_field[ + "row_ignore_symbol" + ] + + # TODO: Account for differences in valid field names with base_tag + if "base_tag" in tag_info: + base_tag = to_tag(tag_info["base_tag"]) + if base_tag in valid_column_names: + valid_column_names[tag_name] = valid_column_names[base_tag] + discard_if_empty.append(tag_name) + if base_tag in row_comment_chars: + row_comment_chars[tag_name] = row_comment_chars[base_tag] + + return valid_column_names, row_comment_chars, discard_if_empty @staticmethod def _read_veda_attr_defaults( diff --git a/times_reader/transforms.py b/times_reader/transforms.py index 66dc155..14db425 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -138,37 +138,8 @@ def validate_input_tables( remove empty tables (for recognized tags). """ - check_list = [ - datatypes.Tag.comagg, - datatypes.Tag.comemi, - datatypes.Tag.currencies, - datatypes.Tag.fi_comm, - datatypes.Tag.fi_process, - datatypes.Tag.fi_t, - datatypes.Tag.tfm_ava, - datatypes.Tag.tfm_comgrp, - datatypes.Tag.tfm_csets, - datatypes.Tag.tfm_dins, - datatypes.Tag.tfm_dins_at, - datatypes.Tag.tfm_dins_ts, - datatypes.Tag.tfm_dins_tsl, - datatypes.Tag.tfm_ins, - datatypes.Tag.tfm_ins_at, - datatypes.Tag.tfm_ins_ts, - datatypes.Tag.tfm_ins_tsl, - datatypes.Tag.tfm_ins_txt, - datatypes.Tag.tfm_mig, - datatypes.Tag.tfm_psets, - datatypes.Tag.tfm_topdins, - datatypes.Tag.tfm_topins, - datatypes.Tag.tfm_upd, - datatypes.Tag.tfm_upd_at, - datatypes.Tag.tfm_upd_ts, - datatypes.Tag.uc_t, - ] - def discard(table): - if table.tag in check_list: + if table.tag in config.discard_if_empty: return not table.dataframe.shape[0] elif table.tag == datatypes.Tag.unitconversion: print("Dropping ~UNITCONVERSION table") @@ -244,40 +215,6 @@ def normalize_column_aliases( return tables -def apply_postnormalisation_fixes( - config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable] -) -> List[datatypes.EmbeddedXlTable]: - rename_cols_dict = { - datatypes.Tag.comemi: {"commname": "commodity"}, - datatypes.Tag.fi_comm: {"commname": "commodity"}, - datatypes.Tag.fi_process: {"techname": "process"}, - datatypes.Tag.tfm_comgrp: {"value": "allregions"}, - datatypes.Tag.tfm_dins: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_ts: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_tsl: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_ts: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_tsl: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_txt: {"curr": "currency"}, - datatypes.Tag.tfm_mig: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_topdins: {"value": "allregions"}, - datatypes.Tag.tfm_topins: {"value": "allregions"}, - datatypes.Tag.tfm_upd: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_upd_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_upd_ts: {"curr": "currency", "value": "allregions"}, - } - - for table in tables: - if table.tag in rename_cols_dict: - table.dataframe = table.dataframe.rename( - columns=rename_cols_dict[table.tag] - ) - - return tables - - def include_tables_source( config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable] ) -> List[datatypes.EmbeddedXlTable]: