From 8869a687006b477527d4b7537e11ead46dd710bf Mon Sep 17 00:00:00 2001 From: Olexandr Balyk Date: Tue, 5 Dec 2023 20:50:11 -0500 Subject: [PATCH] Simplify normalisation (#147) This PR: - restructures veda-tags.json - removes apply_postnormalisation_fixes - moves info on which tables can be discarded if empty to `config` --------- Co-authored-by: Siddharth Krishna --- times_reader/__main__.py | 1 - times_reader/config/veda-tags.json | 3751 +++++++++++++++++----------- times_reader/datatypes.py | 64 +- times_reader/transforms.py | 65 +- 4 files changed, 2311 insertions(+), 1570 deletions(-) diff --git a/times_reader/__main__.py b/times_reader/__main__.py index acb1f72..ffba993 100644 --- a/times_reader/__main__.py +++ b/times_reader/__main__.py @@ -63,7 +63,6 @@ def convert_xl_to_times( lambda config, tables: [ transforms.remove_comment_rows(config, t) for t in tables ], - transforms.apply_postnormalisation_fixes, transforms.generate_dummy_processes, transforms.process_transform_insert_variants, transforms.process_transform_insert, diff --git a/times_reader/config/veda-tags.json b/times_reader/config/veda-tags.json index 0a2fe5a..70b51a4 100644 --- a/times_reader/config/veda-tags.json +++ b/times_reader/config/veda-tags.json @@ -10,20 +10,20 @@ "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "bookname", - "region" - ], - "fields_aliases": [ - [], - [] - ], - "row_ignore_symbol": [ - [], - [] - ] - } + "valid_fields": [ + { + "name": "bookname", + "aliases": [], + "use_name": "bookname", + "row_ignore_symbol": [] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [] + } + ] }, { "tag_name": "comemi", @@ -31,22 +31,19 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "commname" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "commname", + "aliases": [ "commodity" - ] - ], - "row_ignore_symbol": [ - [ + ], + "use_name": "commodity", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "currencies", @@ -79,80 +76,101 @@ "SubRES", "SysSettings" ], - "tag_fields": { - "fields_names": [ - "commdesc", - "commname", - "csets", - "ctslvl", - "ctype", - "limtype", - "peakts", - "region", - "unit" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "commdesc", + "aliases": [ "description" ], - [ + "use_name": "commdesc", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "commname", + "aliases": [ "commodity" ], - [ + "use_name": "commodity", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "csets", + "aliases": [ "cset", "sets", "set" ], - [ + "use_name": "csets", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "ctslvl", + "aliases": [ "tslvl", "timeslicelevel" ], - [ + "use_name": "ctslvl", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "ctype", + "aliases": [ "type", "commoditytype" ], - [ + "use_name": "ctype", + "row_ignore_symbol": [] + }, + { + "name": "limtype", + "aliases": [ "bd" ], - [ - "peaktimeslice" - ], - [], - [] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [ + "use_name": "limtype", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "peakts", + "aliases": [ + "peaktimeslice" ], - [], - [ + "use_name": "peakts", + "row_ignore_symbol": [] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "unit", + "aliases": [], + "use_name": "unit", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "fi_process", @@ -160,83 +178,104 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "primarycg", - "region", - "sets", - "tact", - "tcap", - "techdesc", - "techname", - "tslvl", - "vintage" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "primarycg", + "aliases": [ "pcg", "primarycommoditygroup", "primary_commodity_group" ], - [], - [ - "set" - ], - [ - "activityunit", - "activity_unit" - ], - [ - "capacityunit", - "capacity_unit" - ], - [ - "description" - ], - [ - "process" - ], - [ - "timeslicelevel" - ], - [] - ], - "row_ignore_symbol": [ - [ + "use_name": "primarycg", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "sets", + "aliases": [ + "set" ], - [ + "use_name": "sets", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tact", + "aliases": [ + "activityunit", + "activity_unit" ], - [ + "use_name": "tact", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tcap", + "aliases": [ + "capacityunit", + "capacity_unit" ], - [ + "use_name": "tcap", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "techdesc", + "aliases": [ + "description" ], - [ + "use_name": "techdesc", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "techname", + "aliases": [ + "process" ], - [ + "use_name": "process", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "tslvl", + "aliases": [ + "timeslicelevel" ], - [ + "use_name": "tslvl", + "row_ignore_symbol": [ "\\I:", "*" - ], - [] - ] - } + ] + }, + { + "name": "vintage", + "aliases": [], + "use_name": "vintage", + "row_ignore_symbol": [] + } + ] }, { "tag_name": "fi_t", @@ -244,43 +283,49 @@ "BY", "SubRES" ], - "tag_fields": { - "fields_names": [ - "attribute", - "commodity", - "commodity-in", - "commodity-in-aux", - "commodity-out", - "commodity-out-aux", - "currency", - "limtype", - "other_indexes", - "process", - "region", - "sow", - "stage", - "timeslice", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attribute", + "aliases": [ "parameter", "prmtr", "attr", "attrib" ], - [ + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "commodity", + "aliases": [ "commname", "cset_cn" ], - [ + "use_name": "commodity", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-in", + "aliases": [ "comm-in", "comm_in", "commin", "commodity_in", "commodityin" ], - [ + "use_name": "commodity-in", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-in-aux", + "aliases": [ "comm-in-a", "comm_in-a", "commin-a", @@ -288,14 +333,28 @@ "commodityin-aux", "commodity_in_aux" ], - [ + "use_name": "commodity-in-aux", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-out", + "aliases": [ "comm-out", "comm_out", "commout", "commodity_out", "commodityout" ], - [ + "use_name": "commodity-out", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "commodity-out-aux", + "aliases": [ "comm-out-a", "comm_out-a", "commout-a", @@ -303,93 +362,112 @@ "commodityout-aux", "commodity_out_aux" ], - [ + "use_name": "commodity-out-aux", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "currency", + "aliases": [ "curr" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind", "commgrp", "commodity_group" ], - [ - "techname", - "pset_pn" - ], - [], - [], - [], - [ - "timeslices", - "time_slice", - "time_slices", - "ts" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "process", + "aliases": [ + "techname", + "pset_pn" ], - [ + "use_name": "process", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice", + "aliases": [ + "timeslices", + "time_slice", + "time_slices", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "milestoneyears", @@ -408,200 +486,254 @@ "tag_allowed_in": [ "SR_Trans" ], - "tag_fields": { - "fields_names": [ - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_comgrp", "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "cset_cd", - "cset_cn", - "cset_set", - "description", - "name", - "value" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [], - [], - [ - "valfield", - "val_field", - "allregions" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "description", + "aliases": [], + "use_name": "description", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "name", + "aliases": [], + "use_name": "name", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_csets", "tag_allowed_in": [ "setrules" ], - "tag_fields": { - "fields_names": [ - "and_or_negative_dimension", - "and_or_negative_for_sets", - "and_or_positive_dimension", - "and_or_positive_for_sets", - "comment", - "description", - "dimension_name", - "dimension_set_name", - "input_commodity", - "name", - "outout_commodity", - "set_description", - "set_name" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "and_or_negative_dimension", + "aliases": [ "c_neg_andor" ], - [ + "use_name": "and_or_negative_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_negative_for_sets", + "aliases": [ "c_neg_andor_forsets" ], - [ + "use_name": "and_or_negative_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_dimension", + "aliases": [ "c_pos_andor" ], - [ + "use_name": "and_or_positive_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_for_sets", + "aliases": [ "c_pos_andor_forsets" ], - [ + "use_name": "and_or_positive_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "comment", + "aliases": [ "comments" ], - [ + "use_name": "comment", + "row_ignore_symbol": [] + }, + { + "name": "description", + "aliases": [ "cset_cd" ], - [], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [] + }, + { + "name": "dimension_name", + "aliases": [], + "use_name": "dimension_name", + "row_ignore_symbol": [] + }, + { + "name": "dimension_set_name", + "aliases": [ "cset_set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [] + }, + { + "name": "input_commodity", + "aliases": [ "pset_ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [] + }, + { + "name": "name", + "aliases": [ "cset_cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [] + }, + { + "name": "outout_commodity", + "aliases": [ "pset_co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [] + }, + { + "name": "set_description", + "aliases": [ "setdesc", "set_desc" ], - [ + "use_name": "set_description", + "row_ignore_symbol": [] + }, + { + "name": "set_name", + "aliases": [ "setname" + ], + "use_name": "set_name", + "row_ignore_symbol": [ + "\\I:", + "*" ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "\\I:", - "*" - ] - ] - } + } + ] }, { "tag_name": "tfm_dins", @@ -612,128 +744,167 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attribute", - "cset_cn", - "curr", - "limtype", - "other_indexes", - "pset_pn", - "region", - "side", - "sow", - "stage", - "timeslice", - "ts_filter", - "uc_n", - "value", - "year" - ], - "fields_aliases": [ - [], - [ + "valid_fields": [ + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [], - [ - "user_constraint" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "ts_filter", + "aliases": [], + "use_name": "ts_filter", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_dins-at", @@ -781,239 +952,335 @@ "ParScen", "SysSettings" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "operation_sum_avg_count", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "sourcescen", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "operation_sum_avg_count", + "aliases": [], + "use_name": "operation_sum_avg_count", + "row_ignore_symbol": [] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [ - "scenario", - "scenario name", - "scen" - ], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "sourcescen", + "aliases": [ + "scenario", + "scenario name", + "scen" ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_ins", @@ -1024,243 +1291,342 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "ts_filter", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [], - [], - [], - [], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [ + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "ts_filter", + "aliases": [], + "use_name": "ts_filter", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_ins-at", @@ -1307,397 +1673,556 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "attribute2", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "commodity2", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "currency2", - "limtype", - "limtype2", - "other_indexes", - "other_indexes2", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sourcescen", - "sow", - "sow2", - "stage", - "stage2", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "timeslice2", - "top_check", - "uc_n", - "val_cond", - "value", - "year", - "year2" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute2", + "aliases": [], + "use_name": "attribute2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "commodity2", + "aliases": [ "cset_cn2", "commname2", "comm_name2", "cset:cn2", "cset: cn2" ], - [ + "use_name": "commodity2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "currency2", + "aliases": [ "curr2" ], - [ + "use_name": "currency2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype2", + "aliases": [ "lim_type2", "bd2" ], - [ + "use_name": "limtype2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes2", + "aliases": [ "other_ind2", "oth_ind2" ], - [ + "use_name": "other_indexes2", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "time_slice2", - "ts2" - ], - [ - "top_chk" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ], - [ - "yr2" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:", - "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sourcescen", + "aliases": [], + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "sow2", + "aliases": [], + "use_name": "sow2", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ + ] + }, + { + "name": "stage2", + "aliases": [], + "use_name": "stage2", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "timeslice2", + "aliases": [ + "time_slice2", + "ts2" ], - [], - [], - [], - [], - [ + "use_name": "timeslice2", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "year2", + "aliases": [ + "yr2" ], - [ + "use_name": "year2", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_psets", "tag_allowed_in": [ "setrules" ], - "tag_fields": { - "fields_names": [ - "and_or_negative_dimension", - "and_or_negative_for_sets", - "and_or_positive_dimension", - "and_or_positive_for_sets", - "comment", - "description", - "dimension_name", - "dimension_set_name", - "input_commodity", - "name", - "outout_commodity", - "set_description", - "set_name" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "and_or_negative_dimension", + "aliases": [ "t_neg_andor" ], - [ + "use_name": "and_or_negative_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_negative_for_sets", + "aliases": [ "t_neg_andor_forsets" ], - [ + "use_name": "and_or_negative_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_dimension", + "aliases": [ "t_pos_andor" ], - [ + "use_name": "and_or_positive_dimension", + "row_ignore_symbol": [] + }, + { + "name": "and_or_positive_for_sets", + "aliases": [ "t_pos_andor_forsets" ], - [ + "use_name": "and_or_positive_for_sets", + "row_ignore_symbol": [] + }, + { + "name": "comment", + "aliases": [ "comments" ], - [ + "use_name": "comment", + "row_ignore_symbol": [] + }, + { + "name": "description", + "aliases": [ "pset_pd" ], - [], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [] + }, + { + "name": "dimension_name", + "aliases": [], + "use_name": "dimension_name", + "row_ignore_symbol": [] + }, + { + "name": "dimension_set_name", + "aliases": [ "pset_set" ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [] + }, + { + "name": "input_commodity", + "aliases": [ "pset_ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [] + }, + { + "name": "name", + "aliases": [ "pset_pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [] + }, + { + "name": "outout_commodity", + "aliases": [ "pset_co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [] + }, + { + "name": "set_description", + "aliases": [ "setdesc", "set_desc" ], - [ + "use_name": "set_description", + "row_ignore_symbol": [] + }, + { + "name": "set_name", + "aliases": [ "setname" + ], + "use_name": "set_name", + "row_ignore_symbol": [ + "\\I:", + "*" ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "\\I:", - "*" - ] - ] - } + } + ] }, { "tag_name": "tfm_topdins", @@ -1708,52 +2233,58 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "cset_cn", - "pset_pn", - "region", - "value" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "value", + "aliases": [ "valfield", "val_field", "allregions", "io" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:", - "*" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_topins", @@ -1764,123 +2295,171 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "value" - ], - "fields_aliases": [ - [], - [], - [], - [], - [ + "valid_fields": [ + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "value", + "aliases": [ "valfield", "val_field", "allregions", "io" - ] - ], - "row_ignore_symbol": [ - [], - [], - [], - [], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [], - [], - [], - [], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" ] - ] - } + } + ] }, { "tag_name": "tfm_upd", @@ -1891,243 +2470,342 @@ "RegScen", "TradeScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "curr", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sourcescen", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [], + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "curr", + "aliases": [ "currency" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sourcescen", + "aliases": [], + "use_name": "sourcescen", + "row_ignore_symbol": [ "\\I:" - ], - [ + ] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "tfm_upd-at", @@ -2165,286 +2843,391 @@ "SysSettings", "RegScen" ], - "tag_fields": { - "fields_names": [ - "attrib_cond", - "attribute", - "avc_limtype", - "avc_timeslice", - "avc_year", - "c_neg_andor", - "c_neg_andor_forsets", - "c_pos_andor", - "c_pos_andor_forsets", - "cset_cd", - "cset_cn", - "cset_set", - "currency", - "limtype", - "other_indexes", - "pset_ci", - "pset_co", - "pset_pd", - "pset_pn", - "pset_set", - "region", - "side", - "sow", - "stage", - "t_neg_andor", - "t_neg_andor_forsets", - "t_pos_andor", - "t_pos_andor_forsets", - "timeslice", - "top_check", - "uc_attr", - "uc_desc", - "uc_n", - "val_cond", - "value", - "year" - ], - "fields_aliases": [ - [ + "valid_fields": [ + { + "name": "attrib_cond", + "aliases": [ "attribute_condition" ], - [ + "use_name": "attrib_cond", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "attribute", + "aliases": [ "prmtr", "parameter" ], - [ + "use_name": "attribute", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_limtype", + "aliases": [ "avc_lim_type" ], - [ + "use_name": "avc_limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_timeslice", + "aliases": [ "avc_time_slice" ], - [], - [], - [], - [], - [], - [ + "use_name": "avc_timeslice", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "avc_year", + "aliases": [], + "use_name": "avc_year", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "c_neg_andor", + "aliases": [], + "use_name": "c_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_neg_andor_forsets", + "aliases": [], + "use_name": "c_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor", + "aliases": [], + "use_name": "c_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "c_pos_andor_forsets", + "aliases": [], + "use_name": "c_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "cset_cd", + "aliases": [ "cset:cd", "cset: cd" ], - [ + "use_name": "cset_cd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_cn", + "aliases": [ "commodity", "commname", "comm_name", "cset:cn", "cset: cn" ], - [ + "use_name": "cset_cn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "cset_set", + "aliases": [ "cset:set", "cset: set" ], - [ + "use_name": "cset_set", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "currency", + "aliases": [ "curr" ], - [ + "use_name": "currency", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "limtype", + "aliases": [ "lim_type", "bd" ], - [ + "use_name": "limtype", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "other_indexes", + "aliases": [ "other_ind", "oth_ind" ], - [ + "use_name": "other_indexes", + "row_ignore_symbol": [ + "\\I:", + "*" + ] + }, + { + "name": "pset_ci", + "aliases": [ "pset:ci", "pset: ci" ], - [ + "use_name": "pset_ci", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_co", + "aliases": [ "pset:co", "pset: co" ], - [ + "use_name": "pset_co", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pd", + "aliases": [ "pset:pd", "pset: pd" ], - [ + "use_name": "pset_pd", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_pn", + "aliases": [ "process", "techname", "tech_name", "pset:pn", "pset: pn" ], - [ + "use_name": "pset_pn", + "row_ignore_symbol": [ + "\\I:" + ] + }, + { + "name": "pset_set", + "aliases": [ "pset:set", "pset: set" ], - [], - [], - [], - [], - [], - [], - [], - [], - [ - "time_slice", - "ts" - ], - [ - "top_chk" - ], - [ - "uc-attr", - "ucattr" - ], - [ - "description", - "uc_description" - ], - [ - "user_constraint" - ], - [ - "value_condition" - ], - [ - "valfield", - "val_field", - "allregions" - ], - [ - "yr" - ] - ], - "row_ignore_symbol": [ - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [], - [], - [], - [], - [ + "use_name": "pset_set", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:" - ], - [ + ] + }, + { + "name": "region", + "aliases": [], + "use_name": "region", + "row_ignore_symbol": [ "\\I:" - ], - [ - "\\I:", - "*" - ], - [ - "\\I:", - "*" - ], - [ + ] + }, + { + "name": "side", + "aliases": [], + "use_name": "side", + "row_ignore_symbol": [] + }, + { + "name": "sow", + "aliases": [], + "use_name": "sow", + "row_ignore_symbol": [ "\\I:", "*" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [ - "\\I:" - ], - [], - [ + ] + }, + { + "name": "stage", + "aliases": [], + "use_name": "stage", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "t_neg_andor", + "aliases": [], + "use_name": "t_neg_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_neg_andor_forsets", + "aliases": [], + "use_name": "t_neg_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor", + "aliases": [], + "use_name": "t_pos_andor", + "row_ignore_symbol": [] + }, + { + "name": "t_pos_andor_forsets", + "aliases": [], + "use_name": "t_pos_andor_forsets", + "row_ignore_symbol": [] + }, + { + "name": "timeslice", + "aliases": [ + "time_slice", + "ts" ], - [ + "use_name": "timeslice", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "top_check", + "aliases": [ + "top_chk" ], - [], - [], - [], - [], - [ - "\\I:", - "*" + "use_name": "top_check", + "row_ignore_symbol": [] + }, + { + "name": "uc_attr", + "aliases": [ + "uc-attr", + "ucattr" ], - [], - [ + "use_name": "uc_attr", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_desc", + "aliases": [ + "description", + "uc_description" ], - [ + "use_name": "uc_desc", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "uc_n", + "aliases": [ + "user_constraint" ], - [ + "use_name": "uc_n", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "val_cond", + "aliases": [ + "value_condition" ], - [ + "use_name": "val_cond", + "row_ignore_symbol": [ "\\I:", "*" + ] + }, + { + "name": "value", + "aliases": [ + "valfield", + "val_field", + "allregions" ], - [ + "use_name": "allregions", + "row_ignore_symbol": [ "\\I:" + ] + }, + { + "name": "year", + "aliases": [ + "yr" ], - [ + "use_name": "year", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] }, { "tag_name": "unitconversion", "tag_allowed_in": [ "SysSettings" ], - "tag_fields": { - "fields_names": [ - "from_unit", - "multiplier", - "to_unit" - ], - "fields_aliases": [ - [], - [], - [] - ], - "row_ignore_symbol": [ - [ + "valid_fields": [ + { + "name": "from_unit", + "aliases": [], + "use_name": "from_unit", + "row_ignore_symbol": [ "\\I:", "*" - ], - [], - [ + ] + }, + { + "name": "multiplier", + "aliases": [], + "use_name": "multiplier", + "row_ignore_symbol": [] + }, + { + "name": "to_unit", + "aliases": [], + "use_name": "to_unit", + "row_ignore_symbol": [ "\\I:", "*" ] - ] - } + } + ] } ] diff --git a/times_reader/datatypes.py b/times_reader/datatypes.py index 5d23f33..d5659db 100644 --- a/times_reader/datatypes.py +++ b/times_reader/datatypes.py @@ -151,6 +151,8 @@ class Config: column_aliases: Dict[Tag, Dict[str, str]] # For each tag, this dictionary specifies comment row symbols by column name row_comment_chars: Dict[Tag, Dict[str, list]] + # List of tags for which empty tables should be discarded + discard_if_empty: Iterable[Tag] veda_attr_defaults: Dict[str, Dict[str, list]] def __init__( @@ -164,9 +166,11 @@ def __init__( self.dd_table_order, self.all_attributes = Config._process_times_info( times_info_file ) - self.column_aliases, self.row_comment_chars = Config._read_veda_tags_info( - veda_tags_file - ) + ( + self.column_aliases, + self.row_comment_chars, + self.discard_if_empty, + ) = Config._read_veda_tags_info(veda_tags_file) self.veda_attr_defaults, self.attr_aliases = Config._read_veda_attr_defaults( veda_attr_defaults_file ) @@ -270,7 +274,7 @@ def _read_mappings(filename: str) -> List[TimesXlMap]: @staticmethod def _read_veda_tags_info( veda_tags_file: str, - ) -> Tuple[Dict[Tag, Dict[str, str]], Dict[Tag, Dict[str, list]]]: + ) -> Tuple[Dict[Tag, Dict[str, str]], Dict[Tag, Dict[str, list]], Iterable[Tag]]: def to_tag(s: str) -> Tag: # The file stores the tag name in lowercase, and without the ~ return Tag("~" + s.upper()) @@ -287,27 +291,45 @@ def to_tag(s: str) -> Tag: f"WARNING: datatypes.Tag has an unknown Tag {tag} not in {veda_tags_file}" ) - column_aliases = {} + valid_column_names = {} row_comment_chars = {} + discard_if_empty = [] for tag_info in veda_tags_info: - if "tag_fields" in tag_info: - tag_name = to_tag(tag_info["tag_name"]) - # Process column aliases: - column_aliases[tag_name] = {} - names = tag_info["tag_fields"]["fields_names"] - aliases = tag_info["tag_fields"]["fields_aliases"] - assert len(names) == len(aliases) - for name, aliases in zip(names, aliases): - for alias in aliases: - column_aliases[tag_name][alias] = name - # Process comment chars: + tag_name = to_tag(tag_info["tag_name"]) + if "valid_fields" in tag_info: + discard_if_empty.append(tag_name) + + valid_column_names[tag_name] = {} row_comment_chars[tag_name] = {} - chars = tag_info["tag_fields"]["row_ignore_symbol"] - assert len(names) == len(chars) - for name, chars_list in zip(names, chars): - row_comment_chars[tag_name][name] = chars_list - return column_aliases, row_comment_chars + # Process column aliases and comment chars: + for valid_field in tag_info["valid_fields"]: + valid_field_names = valid_field["aliases"] + if ( + "use_name" in valid_field + and valid_field["use_name"] != valid_field["name"] + ): + field_name = valid_field["use_name"] + valid_field_names.append(valid_field["name"]) + else: + field_name = valid_field["name"] + + for valid_field_name in valid_field_names: + valid_column_names[tag_name][valid_field_name] = field_name + row_comment_chars[tag_name][field_name] = valid_field[ + "row_ignore_symbol" + ] + + # TODO: Account for differences in valid field names with base_tag + if "base_tag" in tag_info: + base_tag = to_tag(tag_info["base_tag"]) + if base_tag in valid_column_names: + valid_column_names[tag_name] = valid_column_names[base_tag] + discard_if_empty.append(tag_name) + if base_tag in row_comment_chars: + row_comment_chars[tag_name] = row_comment_chars[base_tag] + + return valid_column_names, row_comment_chars, discard_if_empty @staticmethod def _read_veda_attr_defaults( diff --git a/times_reader/transforms.py b/times_reader/transforms.py index 66dc155..14db425 100644 --- a/times_reader/transforms.py +++ b/times_reader/transforms.py @@ -138,37 +138,8 @@ def validate_input_tables( remove empty tables (for recognized tags). """ - check_list = [ - datatypes.Tag.comagg, - datatypes.Tag.comemi, - datatypes.Tag.currencies, - datatypes.Tag.fi_comm, - datatypes.Tag.fi_process, - datatypes.Tag.fi_t, - datatypes.Tag.tfm_ava, - datatypes.Tag.tfm_comgrp, - datatypes.Tag.tfm_csets, - datatypes.Tag.tfm_dins, - datatypes.Tag.tfm_dins_at, - datatypes.Tag.tfm_dins_ts, - datatypes.Tag.tfm_dins_tsl, - datatypes.Tag.tfm_ins, - datatypes.Tag.tfm_ins_at, - datatypes.Tag.tfm_ins_ts, - datatypes.Tag.tfm_ins_tsl, - datatypes.Tag.tfm_ins_txt, - datatypes.Tag.tfm_mig, - datatypes.Tag.tfm_psets, - datatypes.Tag.tfm_topdins, - datatypes.Tag.tfm_topins, - datatypes.Tag.tfm_upd, - datatypes.Tag.tfm_upd_at, - datatypes.Tag.tfm_upd_ts, - datatypes.Tag.uc_t, - ] - def discard(table): - if table.tag in check_list: + if table.tag in config.discard_if_empty: return not table.dataframe.shape[0] elif table.tag == datatypes.Tag.unitconversion: print("Dropping ~UNITCONVERSION table") @@ -244,40 +215,6 @@ def normalize_column_aliases( return tables -def apply_postnormalisation_fixes( - config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable] -) -> List[datatypes.EmbeddedXlTable]: - rename_cols_dict = { - datatypes.Tag.comemi: {"commname": "commodity"}, - datatypes.Tag.fi_comm: {"commname": "commodity"}, - datatypes.Tag.fi_process: {"techname": "process"}, - datatypes.Tag.tfm_comgrp: {"value": "allregions"}, - datatypes.Tag.tfm_dins: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_ts: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_dins_tsl: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_ts: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_tsl: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_ins_txt: {"curr": "currency"}, - datatypes.Tag.tfm_mig: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_topdins: {"value": "allregions"}, - datatypes.Tag.tfm_topins: {"value": "allregions"}, - datatypes.Tag.tfm_upd: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_upd_at: {"curr": "currency", "value": "allregions"}, - datatypes.Tag.tfm_upd_ts: {"curr": "currency", "value": "allregions"}, - } - - for table in tables: - if table.tag in rename_cols_dict: - table.dataframe = table.dataframe.rename( - columns=rename_cols_dict[table.tag] - ) - - return tables - - def include_tables_source( config: datatypes.Config, tables: List[datatypes.EmbeddedXlTable] ) -> List[datatypes.EmbeddedXlTable]: