Skip to content

Commit

Permalink
Fixing data packaging
Browse files Browse the repository at this point in the history
  • Loading branch information
@romainsacchi
romainsacchi committed Dec 21, 2024
1 parent 8678c82 commit f472809
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 46 deletions.
30 changes: 12 additions & 18 deletions premise/external.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def _update_external_scenarios(

scenario["database"] = checked_database
scenario["database"].extend(checked_inventories)
configurations = dictionary_merge(configurations, configuration)
configurations[d] = configuration

external_scenario = ExternalScenario(
database=scenario["database"],
Expand All @@ -109,7 +109,7 @@ def _update_external_scenarios(
external_scenarios_data=scenario["external data"],
version=version,
system_model=system_model,
# configurations=configurations,
configurations=configurations,
)
external_scenario.create_markets()
external_scenario.relink_datasets()
Expand Down Expand Up @@ -433,18 +433,16 @@ def __init__(
for v in list(value):
self.fuel_map_reverse[v] = key

external_scenario_regions = []
external_scenario_regions = {}
for datapackage_number, datapackage in enumerate(self.datapackages):
external_scenario_regions.extend(
self.external_scenarios_data[datapackage_number]["regions"]
)
external_scenario_regions[datapackage_number] = self.external_scenarios_data[datapackage_number]["regions"]

self.configurations = configurations or {}
self.configurations = configurations or []

ds_names = get_recursively(self.configurations, "name")

for data in self.external_scenarios_data.values():
self.regionalize_inventories(ds_names, external_scenario_regions, data)
for d, data in self.external_scenarios_data.items():
ds_names = get_recursively(self.configurations[d], "name")
self.regionalize_inventories(ds_names, external_scenario_regions[d], data)

self.dict_bio_flows = get_biosphere_flow_uuid(self.version)
self.outdated_flows = get_correspondence_bio_flows()
Expand All @@ -463,9 +461,10 @@ def regionalize_inventories(self, ds_names, regions, data: dict) -> None:
ws.equals("regionalize", True),
ws.either(*[ws.contains("name", name) for name in list(set(ds_names))]),
):
# remove "regionalize" tag
if "regionalize" in ds:
del ds["regionalize"]
if ds.get("regionalize", False) is False:
continue

del ds["regionalize"]

processed_key = (ds["name"], ds["reference product"], ds["unit"])
if ds["location"] not in regions and processed_key not in processed:
Expand All @@ -491,11 +490,6 @@ def regionalize_inventories(self, ds_names, regions, data: dict) -> None:
self.year
in data["production volume"].coords["year"].values
):
print(region)
print(ds["production volume variable"])
print(
data["production volume"].coords["variables"].values
)
act["production volume"] = (
data["production volume"]
.sel(
Expand Down
50 changes: 22 additions & 28 deletions premise/pathways.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,45 +292,39 @@ def add_scenario_data(self):
"""
# concatenate xarray across IAM scenarios
scenario_data = [
self.datapackage.scenarios[s]["iam data"].data
for s in range(len(self.scenarios))
]

# add scenario data from external scenarios
extra_units = {}
data_list, extra_units = [], {}
for scenario in self.datapackage.scenarios:
data = scenario["iam data"].production_volumes.interp(year=scenario["year"])
extra_units.update(scenario["iam data"].production_volumes.attrs["unit"])
scenario_name = f"{scenario['model']} - {scenario['pathway']}"
if "external data" in scenario:
for s in scenario["external data"].values():
scenario_data.append(
s["production volume"].interp(
year=self.years,
kwargs={"fill_value": "extrapolate"},
)
for ext, external in scenario["external data"].items():
data = xr.concat(
[
data,
external["production volume"].interp(
year=scenario["year"]
),
],
dim="variables",
)
extra_units.update(s["production volume"].attrs["unit"])
extra_units.update(external["production volume"].attrs["unit"])
scenario_name += f" - {scenario['external scenarios'][ext]['scenario']}"

array = xr.concat(scenario_data, dim="scenario")
# add a scenario dimension
data = data.expand_dims("scenario")
data.coords["scenario"] = [scenario_name]
self.scenario_names.append(scenario_name)

print(array.coords)
print(array.shape)
data_list.append(data)

# add scenario data to the xarray
for s, scenario in enumerate(self.datapackage.scenarios):
name = f"{scenario['model'].upper()} - {scenario['pathway']}"
if "external scenarios" in scenario:
for external in scenario["external scenarios"]:
name += f"-{external['scenario']}"
# self.scenario_names.append(name)
self.scenario_names.append(name)

array.coords["scenario"] = self.scenario_names
array = xr.concat(data_list, dim="scenario")

# make sure pathways/scenario_data directory exists
(Path.cwd() / "pathways" / "scenario_data").mkdir(parents=True, exist_ok=True)
# save the xarray as csv
df = array.to_dataframe().reset_index()

extra_units.update(array.attrs["unit"])

# add a unit column
Expand Down Expand Up @@ -426,4 +420,4 @@ def build_datapackage(self, name: str, contributors: list = None):
# zip the folder
shutil.make_archive(name, "zip", str(Path.cwd() / "pathways"))

print(f"Data package saved at {str(Path.cwd() / 'pathways' / f'{name}.zip')}")
print(f"Data package saved at {str(Path.cwd() / f'{name}.zip')}")

0 comments on commit f472809

Please sign in to comment.