Fix issues with disordered activities categories and locations

.gitignore

@@ -14,6 +14,8 @@ data/logs/
 dev/sample/inventories/*
 *.pyc
 
+.gzip
+
 dev/*.json
 dev/*.csv
 dev/*.pickle

dev/act_categories_agg.yaml

@@ -1,6 +1,5 @@
 Aquaculture: Aquaculture
-Architectural and engineering activities and related technical consultancy: Architectural
-  and Engineering Services
+Architectural and engineering activities and related technical consultancy: Architectural and Engineering Services
 Building of ships and floating structures: Shipbuilding and Marine Engineering
 Carbon capture: Carbon Management
 Carbon storage: Carbon Management
@@ -20,16 +19,14 @@ Extraction of natural gas: Petroleum and Natural Gas Extraction
 Extraction of peat: Salt and Other Mineral Extraction
 Extraction of salt: Salt and Other Mineral Extraction
 Finishing of textiles: Textile Manufacturing
-Forging pressing stamping and rollforming of metal powder metallurgy: Metalworking
-  and Forging
+Forging pressing stamping and rollforming of metal powder metallurgy: Metalworking and Forging
 Freight air transport: Freight Transport
 Freight rail transport: Freight Transport
 Freight transport by road: Freight Transport
 Freshwater aquaculture: Aquaculture
 Gathering of nonwood forest products: Agriculture and Crop Production
 Growing of beverage crops: Agriculture and Crop Production
-Growing of cereals except rice leguminous crops and oil seeds: Agriculture and Crop
-  Production
+Growing of cereals except rice leguminous crops and oil seeds: Agriculture and Crop Production
 Growing of citrus fruits: Agriculture and Crop Production
 Growing of fibre crops: Agriculture and Crop Production
 Growing of grapes: Agriculture and Crop Production
@@ -47,18 +44,15 @@ Inland freight water transport: Freight Transport
 Logging: Forestry and Logging
 Maintenance and repair of motor vehicles: Transport Equipment Maintenance and Repair
 Manufacture of agricultural and forestry machinery: Automotive and Machinery Manufacturing
-Manufacture of air and spacecraft and related machinery: Automotive and Machinery
-  Manufacturing
+Manufacture of air and spacecraft and related machinery: Automotive and Machinery Manufacturing
 Manufacture of articles of concrete cement and plaster: Other Manufacturing
 Manufacture of basic chemical: Chemicals and Fertilizers
 Manufacture of basic chemicals: Chemicals and Fertilizers
-Manufacture of basic chemicals fertilizers and nitrogen compounds plastics: Chemicals
-  and Fertilizers
+Manufacture of basic chemicals fertilizers and nitrogen compounds plastics: Chemicals and Fertilizers
 Manufacture of basic iron and steel: Iron and Steel Manufacturing
 Manufacture of basic precious and other nonferrous metals: Iron and Steel Manufacturing
 Manufacture of bicycles and invalid carriages: Automotive and Machinery Manufacturing
-Manufacture of bodies coachwork for motor vehicles manufacture of traile: Automotive
-  and Machinery Manufacturing
+Manufacture of bodies coachwork for motor vehicles manufacture of traile: Automotive and Machinery Manufacturing
 Manufacture of builders carpentry and joinery: Automotive and Machinery Manufacturing
 Manufacture of chemicals and chemical products: Chemicals and Fertilizers
 Manufacture of clay building materials: Other Manufacturing
@@ -69,30 +63,24 @@ Manufacture of consumer electronics: Electrical and Electronic Manufacturing
 Manufacture of dairy products: Food and Beverage Processing
 Manufacture of domestic appliances: Electrical and Electronic Manufacturing
 Manufacture of electric lighting equipment: Electrical and Electronic Manufacturing
-Manufacture of electric motors generators transformers and electricity di: Electrical
-  and Electronic Manufacturing
+Manufacture of electric motors generators transformers and electricity di: Electrical and Electronic Manufacturing
 Manufacture of electronic components and boards: Electrical and Electronic Manufacturing
-Manufacture of engines and turbines except aircraft vehicle and cycle eng: Automotive
-  and Machinery Manufacturing
+Manufacture of engines and turbines except aircraft vehicle and cycle eng: Automotive and Machinery Manufacturing
 Manufacture of fertilizers and nitrogen compounds: Chemicals and Fertilizers
 Manufacture of furniture: Furniture and Wood Products
 Manufacture of gas distribution of gaseous fuels through mains: Energy Distribution
 Manufacture of glass and glass products: Glass and Ceramics
 Manufacture of grain mill products: Food and Beverage Processing
 Manufacture of knitted and crocheted fabrics: Textile Manufacturing
 Manufacture of machinery for metallurgy: Automotive and Machinery Manufacturing
-Manufacture of machinery for mining quarrying and construction: Automotive and Machinery
-  Manufacturing
+Manufacture of machinery for mining quarrying and construction: Automotive and Machinery Manufacturing
 Manufacture of metalforming machinery and machine tools: Metalworking and Forging
 Manufacture of motor vehicles: Automotive and Machinery Manufacturing
 Manufacture of motorcycles: Automotive and Machinery Manufacturing
-Manufacture of office machinery and equipment except computers and periphe: Other
-  Manufacturing
+Manufacture of office machinery and equipment except computers and periphe: Other Manufacturing
 Manufacture of other articles of paper and paperboard: Other Manufacturing
-Manufacture of other electronic and electric wires and cables: Electrical and Electronic
-  Manufacturing
-Manufacture of other fabricated metal products metalworking service activit: Automotive
-  and Machinery Manufacturing
+Manufacture of other electronic and electric wires and cables: Electrical and Electronic Manufacturing
+Manufacture of other fabricated metal products metalworking service activit: Automotive and Machinery Manufacturing
 Manufacture of other generalpurpose machinery: Automotive and Machinery Manufacturing
 Manufacture of other nonmetallic mineral products nec: Other Manufacturing
 Manufacture of other porcelain and ceramic products: Glass and Ceramics
@@ -101,18 +89,15 @@ Manufacture of other rubber products: Rubber and Plastics
 Manufacture of other specialpurpose machinery: Automotive and Machinery Manufacturing
 Manufacture of other textiles nec: Textile Manufacturing
 Manufacture of ovens furnaces and furnace burners: Other Manufacturing
-Manufacture of parts and accessories for motor vehicles: Automotive and Machinery
-  Manufacturing
+Manufacture of parts and accessories for motor vehicles: Automotive and Machinery Manufacturing
 Manufacture of pesticides and other agrochemical products: Chemicals and Fertilizers
-Manufacture of pharmaceuticals medicinal chemical and botanical products: Chemicals
-  and Fertilizers
+Manufacture of pharmaceuticals medicinal chemical and botanical products: Chemicals and Fertilizers
 Manufacture of plastics and synthetic rubber in primary forms: Rubber and Plastics
 Manufacture of plastics products: Rubber and Plastics
 Manufacture of powerdriven hand tools: Other Manufacturing
 Manufacture of prepared animal feeds: Food and Beverage Processing
 Manufacture of prepared meals and dishes: Food and Beverage Processing
-Manufacture of products of wood cork straw and plaiting materials: Furniture and Wood
-  Products
+Manufacture of products of wood cork straw and plaiting materials: Furniture and Wood Products
 Manufacture of refractory products: Other Manufacturing
 Manufacture of starches and starch products: Food and Beverage Processing
 Manufacture of structural metal products: Iron and Steel Manufacturing
@@ -134,8 +119,7 @@ Other passenger land transport: Passenger Land Transport
 Other specialized construction activities: Construction and Civil Engineering
 Passenger air transport: Air Transport
 Passenger rail transport interurban: Rail Transport
-Photocopying document preparation and other specialized office support act: Miscellaneous
-  Services
+Photocopying document preparation and other specialized office support act: Miscellaneous Services
 Plant propagation: Agriculture and Crop Production
 Preparation and spinning of textile fibres: Textile Manufacturing
 Printing: Printing and Publishing
@@ -148,10 +132,8 @@ Raising of other animals: Animal Husbandry
 Raising of poultry: Animal Husbandry
 Raising of sheep and goats: Animal Husbandry
 Raising of swinepigs: Animal Husbandry
-Remediation activities and other waste management services: Remediation and Waste
-  Management
-Repair of transport equipment except motor vehicles: Transport Equipment Maintenance
-  and Repair
+Remediation activities and other waste management services: Remediation and Waste Management
+Repair of transport equipment except motor vehicles: Transport Equipment Maintenance and Repair
 Sea and coastal freight water transport: Freight Transport
 Seed processing for propagation: Agriculture and Crop Production
 Service activities related to printing: Printing and Publishing
@@ -172,6 +154,5 @@ Urban and suburban passenger land transport: Passenger Land Transport
 Washing and dry cleaning of textile and fur products: Miscellaneous Services
 Water collection treatment and supply: Energy Distribution
 Weaving of textiles: Textile Manufacturing
-Wholesale of solid liquid and gaseous fuels and related products: Fuel and Energy
-  Wholesale
+Wholesale of solid liquid and gaseous fuels and related products: Fuel and Energy Wholesale
 Wired telecommunications activities: Telecommunications

dev/timing.py

@@ -3,7 +3,7 @@
 p = Pathways(
     datapackage="remind-SSP2-PkBudg1150-stem-SPS1.zip",
     geography_mapping="geo_mapping_remind.yaml",
-    #activities_mapping="act_categories_agg.yaml",
+    activities_mapping="act_categories_agg.yaml",
 )
 
 vars = [v for v in p.scenarios.coords["variables"].values if v.startswith("FE")]
@@ -15,9 +15,14 @@
     ],
     regions=["CH"],
     scenarios=p.scenarios.pathway.values.tolist(),
-    years=[2020, 2030, 2040, 2050],
+    years=[
+        2020,
+        #2030,
+        #2040,
+        #2050
+    ],
     variables=vars,
-    use_distributions=10,
+    use_distributions=20,
     subshares=True,
 )
 

pathways/lca.py

@@ -6,32 +6,22 @@
 import logging
 import pickle
 import uuid
-from collections import defaultdict
 from pathlib import Path
-from typing import Any, Dict, List, Tuple
+from typing import Dict, List, Tuple
 
 import bw2calc as bc
 import bw_processing as bwp
 import numpy as np
 import pyprind
 import sparse as sp
-from bw2calc import MultiLCA
 from bw2calc.utils import get_datapackage
 from bw_processing import Datapackage
-from numpy import dtype, ndarray
 from premise.geomap import Geomap
 from scipy import sparse
 
-from .filesystem_constants import DIR_CACHED_DB, STATS_DIR, USER_LOGS_DIR
+from .filesystem_constants import DIR_CACHED_DB, USER_LOGS_DIR
 from .lcia import fill_characterization_factors_matrices
-from .stats import (
-    create_mapping_sheet,
-    log_results,
-    log_subshares,
-    log_uncertainty_values,
-    run_GSA_delta,
-    run_GSA_OLS,
-)
+
 from .subshares import (
     adjust_matrix_based_on_shares,
     find_technology_indices,
@@ -130,7 +120,6 @@ def get_lca_matrices(
     :type scenario: str
     :param year: The year of the scenario.
     :type year: int
-    :param remove_infrastructure: Whether to remove infrastructure exchanges from the technosphere matrix.
     :rtype: Tuple[sparse.csr_matrix, sparse.csr_matrix, Dict, Dict, List]
     """
 
@@ -225,8 +214,6 @@ def remove_double_counting(
     Remove double counting from a technosphere matrix by zeroing out the demanded row values
     in all columns, except for those on the diagonal.
     :param technosphere_matrix: bw2calc.LCA object
-    :param demand: dict with demand values
-    :param activities_to_exclude: list of row indices to zero out
     :return: Technosphere matrix with double counting removed
     """
 
@@ -507,7 +494,6 @@ def _calculate_year(args: tuple):
         scenarios,
         reverse_classifications,
         geography_mapping,
-        activities_mapping,
         debug,
         use_distributions,
         shares,
@@ -581,16 +567,21 @@ def _calculate_year(args: tuple):
         technosphere_indices=technosphere_indices,
         group_by=lambda x: classifications.get(x[:3], "unclassified"),
         group_values=lca_results.coords["act_category"].values.tolist(),
-        mapping=activities_mapping,
     )
 
+    # reorder keys of acts_category_idx_dict based on lca_results.coords["act_category"].values
+    acts_category_idx_dict = {k: acts_category_idx_dict[k] for k in lca_results.coords["act_category"].values.tolist()}
+
     acts_location_idx_dict = _group_technosphere_indices(
         technosphere_indices=technosphere_indices,
         group_by=lambda x: x[-1],
         group_values=list(set([x[-1] for x in technosphere_indices.keys()])),
         mapping=geography_mapping,
     )
 
+    # reorder keys of acts_location_idx_dict based on lca_results.coords["location"].values
+    acts_location_idx_dict = {k: acts_location_idx_dict[k] for k in lca_results.coords["location"].values.tolist()}
+
     bar = pyprind.ProgBar(len(regions))
     for region in regions:
         fus, fus_details = create_functional_units(

pathways/pathways.py

@@ -152,10 +152,7 @@ def __init__(
                 yaml.full_load(self.data.get_resource("classifications").raw_read())
             )
 
-        # create a reverse mapping
-        self.reverse_classifications = defaultdict(list)
-        for k, v in self.classifications.items():
-            self.reverse_classifications[v].append(k)
+
 
         self.lca_results = None
         self.lcia_methods = get_lcia_method_names()
@@ -171,9 +168,15 @@ def __init__(
             self.geography_mapping = None
 
         if activities_mapping:
-            self.activities_mapping = load_mapping(activities_mapping)
-        else:
-            self.activities_mapping = None
+            mapping = load_mapping(activities_mapping)
+            for k, v in self.classifications.items():
+                if v in mapping:
+                    self.classifications[k] = mapping[v]
+
+        # create a reverse mapping
+        self.reverse_classifications = defaultdict(list)
+        for k, v in self.classifications.items():
+            self.reverse_classifications[v].append(k)
 
         clean_cache_directory()
 
@@ -413,23 +416,21 @@ def calculate(
             else:
                 self.geography_mapping = {loc: loc for loc in locations}
 
-            if self.activities_mapping:
-                classifications = list(set(list(self.activities_mapping.values())))
-            else:
-                classifications = list(set(list(self.classifications.values())))
-
             self.lca_results = create_lca_results_array(
                 methods=methods,
                 years=years,
                 regions=regions,
                 locations=locations,
                 models=models,
                 scenarios=scenarios,
-                classifications=classifications,
+                classifications=self.classifications,
                 mapping=self.mapping,
                 use_distributions=use_distributions > 0,
             )
 
+            print(self.lca_results.coords["act_category"].values.tolist())
+            print(self.lca_results.coords["location"].values.tolist())
+
         # generate share of sub-technologies
         shares = None
         if subshares is True:
@@ -462,7 +463,6 @@ def calculate(
                         self.scenarios,
                         self.reverse_classifications,
                         self.geography_mapping,
-                        self.activities_mapping,
                         self.debug,
                         use_distributions,
                         shares,

pathways/utils.py

@@ -168,7 +168,7 @@ def create_lca_results_array(
     locations: List[str],
     models: List[str],
     scenarios: List[str],
-    classifications: list,
+    classifications: dict,
     mapping: dict,
     use_distributions: bool = False,
 ) -> xr.DataArray:
@@ -216,7 +216,7 @@ def create_lca_results_array(
 
     # Define the coordinates for the xarray DataArray
     coords = {
-        "act_category": classifications,
+        "act_category": list(set(list(classifications.values()))),
         "variable": list(mapping.keys()),
         "year": years,
         "region": regions,