From 66e051c881838abd46bcafc0d4622d4ce3b53501 Mon Sep 17 00:00:00 2001 From: romainsacchi Date: Wed, 31 Jul 2024 17:54:10 +0200 Subject: [PATCH] Fix issues with disordered activities categories and locations --- .gitignore | 2 ++ dev/act_categories_agg.yaml | 57 +++++++++++++------------------------ dev/timing.py | 11 +++++-- pathways/lca.py | 27 ++++++------------ pathways/pathways.py | 28 +++++++++--------- pathways/utils.py | 4 +-- 6 files changed, 54 insertions(+), 75 deletions(-) diff --git a/.gitignore b/.gitignore index 7976a4b..68b23bf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ data/logs/ dev/sample/inventories/* *.pyc +.gzip + dev/*.json dev/*.csv dev/*.pickle diff --git a/dev/act_categories_agg.yaml b/dev/act_categories_agg.yaml index bfcc680..0ebcce5 100644 --- a/dev/act_categories_agg.yaml +++ b/dev/act_categories_agg.yaml @@ -1,6 +1,5 @@ Aquaculture: Aquaculture -Architectural and engineering activities and related technical consultancy: Architectural - and Engineering Services +Architectural and engineering activities and related technical consultancy: Architectural and Engineering Services Building of ships and floating structures: Shipbuilding and Marine Engineering Carbon capture: Carbon Management Carbon storage: Carbon Management @@ -20,16 +19,14 @@ Extraction of natural gas: Petroleum and Natural Gas Extraction Extraction of peat: Salt and Other Mineral Extraction Extraction of salt: Salt and Other Mineral Extraction Finishing of textiles: Textile Manufacturing -Forging pressing stamping and rollforming of metal powder metallurgy: Metalworking - and Forging +Forging pressing stamping and rollforming of metal powder metallurgy: Metalworking and Forging Freight air transport: Freight Transport Freight rail transport: Freight Transport Freight transport by road: Freight Transport Freshwater aquaculture: Aquaculture Gathering of nonwood forest products: Agriculture and Crop Production Growing of beverage crops: Agriculture and Crop Production -Growing of cereals except rice leguminous crops and oil seeds: Agriculture and Crop - Production +Growing of cereals except rice leguminous crops and oil seeds: Agriculture and Crop Production Growing of citrus fruits: Agriculture and Crop Production Growing of fibre crops: Agriculture and Crop Production Growing of grapes: Agriculture and Crop Production @@ -47,18 +44,15 @@ Inland freight water transport: Freight Transport Logging: Forestry and Logging Maintenance and repair of motor vehicles: Transport Equipment Maintenance and Repair Manufacture of agricultural and forestry machinery: Automotive and Machinery Manufacturing -Manufacture of air and spacecraft and related machinery: Automotive and Machinery - Manufacturing +Manufacture of air and spacecraft and related machinery: Automotive and Machinery Manufacturing Manufacture of articles of concrete cement and plaster: Other Manufacturing Manufacture of basic chemical: Chemicals and Fertilizers Manufacture of basic chemicals: Chemicals and Fertilizers -Manufacture of basic chemicals fertilizers and nitrogen compounds plastics: Chemicals - and Fertilizers +Manufacture of basic chemicals fertilizers and nitrogen compounds plastics: Chemicals and Fertilizers Manufacture of basic iron and steel: Iron and Steel Manufacturing Manufacture of basic precious and other nonferrous metals: Iron and Steel Manufacturing Manufacture of bicycles and invalid carriages: Automotive and Machinery Manufacturing -Manufacture of bodies coachwork for motor vehicles manufacture of traile: Automotive - and Machinery Manufacturing +Manufacture of bodies coachwork for motor vehicles manufacture of traile: Automotive and Machinery Manufacturing Manufacture of builders carpentry and joinery: Automotive and Machinery Manufacturing Manufacture of chemicals and chemical products: Chemicals and Fertilizers Manufacture of clay building materials: Other Manufacturing @@ -69,11 +63,9 @@ Manufacture of consumer electronics: Electrical and Electronic Manufacturing Manufacture of dairy products: Food and Beverage Processing Manufacture of domestic appliances: Electrical and Electronic Manufacturing Manufacture of electric lighting equipment: Electrical and Electronic Manufacturing -Manufacture of electric motors generators transformers and electricity di: Electrical - and Electronic Manufacturing +Manufacture of electric motors generators transformers and electricity di: Electrical and Electronic Manufacturing Manufacture of electronic components and boards: Electrical and Electronic Manufacturing -Manufacture of engines and turbines except aircraft vehicle and cycle eng: Automotive - and Machinery Manufacturing +Manufacture of engines and turbines except aircraft vehicle and cycle eng: Automotive and Machinery Manufacturing Manufacture of fertilizers and nitrogen compounds: Chemicals and Fertilizers Manufacture of furniture: Furniture and Wood Products Manufacture of gas distribution of gaseous fuels through mains: Energy Distribution @@ -81,18 +73,14 @@ Manufacture of glass and glass products: Glass and Ceramics Manufacture of grain mill products: Food and Beverage Processing Manufacture of knitted and crocheted fabrics: Textile Manufacturing Manufacture of machinery for metallurgy: Automotive and Machinery Manufacturing -Manufacture of machinery for mining quarrying and construction: Automotive and Machinery - Manufacturing +Manufacture of machinery for mining quarrying and construction: Automotive and Machinery Manufacturing Manufacture of metalforming machinery and machine tools: Metalworking and Forging Manufacture of motor vehicles: Automotive and Machinery Manufacturing Manufacture of motorcycles: Automotive and Machinery Manufacturing -Manufacture of office machinery and equipment except computers and periphe: Other - Manufacturing +Manufacture of office machinery and equipment except computers and periphe: Other Manufacturing Manufacture of other articles of paper and paperboard: Other Manufacturing -Manufacture of other electronic and electric wires and cables: Electrical and Electronic - Manufacturing -Manufacture of other fabricated metal products metalworking service activit: Automotive - and Machinery Manufacturing +Manufacture of other electronic and electric wires and cables: Electrical and Electronic Manufacturing +Manufacture of other fabricated metal products metalworking service activit: Automotive and Machinery Manufacturing Manufacture of other generalpurpose machinery: Automotive and Machinery Manufacturing Manufacture of other nonmetallic mineral products nec: Other Manufacturing Manufacture of other porcelain and ceramic products: Glass and Ceramics @@ -101,18 +89,15 @@ Manufacture of other rubber products: Rubber and Plastics Manufacture of other specialpurpose machinery: Automotive and Machinery Manufacturing Manufacture of other textiles nec: Textile Manufacturing Manufacture of ovens furnaces and furnace burners: Other Manufacturing -Manufacture of parts and accessories for motor vehicles: Automotive and Machinery - Manufacturing +Manufacture of parts and accessories for motor vehicles: Automotive and Machinery Manufacturing Manufacture of pesticides and other agrochemical products: Chemicals and Fertilizers -Manufacture of pharmaceuticals medicinal chemical and botanical products: Chemicals - and Fertilizers +Manufacture of pharmaceuticals medicinal chemical and botanical products: Chemicals and Fertilizers Manufacture of plastics and synthetic rubber in primary forms: Rubber and Plastics Manufacture of plastics products: Rubber and Plastics Manufacture of powerdriven hand tools: Other Manufacturing Manufacture of prepared animal feeds: Food and Beverage Processing Manufacture of prepared meals and dishes: Food and Beverage Processing -Manufacture of products of wood cork straw and plaiting materials: Furniture and Wood - Products +Manufacture of products of wood cork straw and plaiting materials: Furniture and Wood Products Manufacture of refractory products: Other Manufacturing Manufacture of starches and starch products: Food and Beverage Processing Manufacture of structural metal products: Iron and Steel Manufacturing @@ -134,8 +119,7 @@ Other passenger land transport: Passenger Land Transport Other specialized construction activities: Construction and Civil Engineering Passenger air transport: Air Transport Passenger rail transport interurban: Rail Transport -Photocopying document preparation and other specialized office support act: Miscellaneous - Services +Photocopying document preparation and other specialized office support act: Miscellaneous Services Plant propagation: Agriculture and Crop Production Preparation and spinning of textile fibres: Textile Manufacturing Printing: Printing and Publishing @@ -148,10 +132,8 @@ Raising of other animals: Animal Husbandry Raising of poultry: Animal Husbandry Raising of sheep and goats: Animal Husbandry Raising of swinepigs: Animal Husbandry -Remediation activities and other waste management services: Remediation and Waste - Management -Repair of transport equipment except motor vehicles: Transport Equipment Maintenance - and Repair +Remediation activities and other waste management services: Remediation and Waste Management +Repair of transport equipment except motor vehicles: Transport Equipment Maintenance and Repair Sea and coastal freight water transport: Freight Transport Seed processing for propagation: Agriculture and Crop Production Service activities related to printing: Printing and Publishing @@ -172,6 +154,5 @@ Urban and suburban passenger land transport: Passenger Land Transport Washing and dry cleaning of textile and fur products: Miscellaneous Services Water collection treatment and supply: Energy Distribution Weaving of textiles: Textile Manufacturing -Wholesale of solid liquid and gaseous fuels and related products: Fuel and Energy - Wholesale +Wholesale of solid liquid and gaseous fuels and related products: Fuel and Energy Wholesale Wired telecommunications activities: Telecommunications diff --git a/dev/timing.py b/dev/timing.py index ca56d2f..d6f9802 100644 --- a/dev/timing.py +++ b/dev/timing.py @@ -3,7 +3,7 @@ p = Pathways( datapackage="remind-SSP2-PkBudg1150-stem-SPS1.zip", geography_mapping="geo_mapping_remind.yaml", - #activities_mapping="act_categories_agg.yaml", + activities_mapping="act_categories_agg.yaml", ) vars = [v for v in p.scenarios.coords["variables"].values if v.startswith("FE")] @@ -15,9 +15,14 @@ ], regions=["CH"], scenarios=p.scenarios.pathway.values.tolist(), - years=[2020, 2030, 2040, 2050], + years=[ + 2020, + #2030, + #2040, + #2050 + ], variables=vars, - use_distributions=10, + use_distributions=20, subshares=True, ) diff --git a/pathways/lca.py b/pathways/lca.py index 351145d..ca598bd 100644 --- a/pathways/lca.py +++ b/pathways/lca.py @@ -6,32 +6,22 @@ import logging import pickle import uuid -from collections import defaultdict from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Dict, List, Tuple import bw2calc as bc import bw_processing as bwp import numpy as np import pyprind import sparse as sp -from bw2calc import MultiLCA from bw2calc.utils import get_datapackage from bw_processing import Datapackage -from numpy import dtype, ndarray from premise.geomap import Geomap from scipy import sparse -from .filesystem_constants import DIR_CACHED_DB, STATS_DIR, USER_LOGS_DIR +from .filesystem_constants import DIR_CACHED_DB, USER_LOGS_DIR from .lcia import fill_characterization_factors_matrices -from .stats import ( - create_mapping_sheet, - log_results, - log_subshares, - log_uncertainty_values, - run_GSA_delta, - run_GSA_OLS, -) + from .subshares import ( adjust_matrix_based_on_shares, find_technology_indices, @@ -130,7 +120,6 @@ def get_lca_matrices( :type scenario: str :param year: The year of the scenario. :type year: int - :param remove_infrastructure: Whether to remove infrastructure exchanges from the technosphere matrix. :rtype: Tuple[sparse.csr_matrix, sparse.csr_matrix, Dict, Dict, List] """ @@ -225,8 +214,6 @@ def remove_double_counting( Remove double counting from a technosphere matrix by zeroing out the demanded row values in all columns, except for those on the diagonal. :param technosphere_matrix: bw2calc.LCA object - :param demand: dict with demand values - :param activities_to_exclude: list of row indices to zero out :return: Technosphere matrix with double counting removed """ @@ -507,7 +494,6 @@ def _calculate_year(args: tuple): scenarios, reverse_classifications, geography_mapping, - activities_mapping, debug, use_distributions, shares, @@ -581,9 +567,11 @@ def _calculate_year(args: tuple): technosphere_indices=technosphere_indices, group_by=lambda x: classifications.get(x[:3], "unclassified"), group_values=lca_results.coords["act_category"].values.tolist(), - mapping=activities_mapping, ) + # reorder keys of acts_category_idx_dict based on lca_results.coords["act_category"].values + acts_category_idx_dict = {k: acts_category_idx_dict[k] for k in lca_results.coords["act_category"].values.tolist()} + acts_location_idx_dict = _group_technosphere_indices( technosphere_indices=technosphere_indices, group_by=lambda x: x[-1], @@ -591,6 +579,9 @@ def _calculate_year(args: tuple): mapping=geography_mapping, ) + # reorder keys of acts_location_idx_dict based on lca_results.coords["location"].values + acts_location_idx_dict = {k: acts_location_idx_dict[k] for k in lca_results.coords["location"].values.tolist()} + bar = pyprind.ProgBar(len(regions)) for region in regions: fus, fus_details = create_functional_units( diff --git a/pathways/pathways.py b/pathways/pathways.py index 50564fc..0c61659 100644 --- a/pathways/pathways.py +++ b/pathways/pathways.py @@ -152,10 +152,7 @@ def __init__( yaml.full_load(self.data.get_resource("classifications").raw_read()) ) - # create a reverse mapping - self.reverse_classifications = defaultdict(list) - for k, v in self.classifications.items(): - self.reverse_classifications[v].append(k) + self.lca_results = None self.lcia_methods = get_lcia_method_names() @@ -171,9 +168,15 @@ def __init__( self.geography_mapping = None if activities_mapping: - self.activities_mapping = load_mapping(activities_mapping) - else: - self.activities_mapping = None + mapping = load_mapping(activities_mapping) + for k, v in self.classifications.items(): + if v in mapping: + self.classifications[k] = mapping[v] + + # create a reverse mapping + self.reverse_classifications = defaultdict(list) + for k, v in self.classifications.items(): + self.reverse_classifications[v].append(k) clean_cache_directory() @@ -413,11 +416,6 @@ def calculate( else: self.geography_mapping = {loc: loc for loc in locations} - if self.activities_mapping: - classifications = list(set(list(self.activities_mapping.values()))) - else: - classifications = list(set(list(self.classifications.values()))) - self.lca_results = create_lca_results_array( methods=methods, years=years, @@ -425,11 +423,14 @@ def calculate( locations=locations, models=models, scenarios=scenarios, - classifications=classifications, + classifications=self.classifications, mapping=self.mapping, use_distributions=use_distributions > 0, ) + print(self.lca_results.coords["act_category"].values.tolist()) + print(self.lca_results.coords["location"].values.tolist()) + # generate share of sub-technologies shares = None if subshares is True: @@ -462,7 +463,6 @@ def calculate( self.scenarios, self.reverse_classifications, self.geography_mapping, - self.activities_mapping, self.debug, use_distributions, shares, diff --git a/pathways/utils.py b/pathways/utils.py index bd3733f..38dec13 100644 --- a/pathways/utils.py +++ b/pathways/utils.py @@ -168,7 +168,7 @@ def create_lca_results_array( locations: List[str], models: List[str], scenarios: List[str], - classifications: list, + classifications: dict, mapping: dict, use_distributions: bool = False, ) -> xr.DataArray: @@ -216,7 +216,7 @@ def create_lca_results_array( # Define the coordinates for the xarray DataArray coords = { - "act_category": classifications, + "act_category": list(set(list(classifications.values()))), "variable": list(mapping.keys()), "year": years, "region": regions,