From 23ca6a72c58e8b6ecbcd636016f4a765f97e02ec Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Wed, 18 Sep 2024 15:53:37 +0200 Subject: [PATCH 01/12] added SEALS allocation script --- DESCRIPTION | 2 +- config/default.cfg | 21 +- scripts/output/extra/disaggregation.R | 1 - scripts/output/extra/runSEALSallocation.R | 231 ++++++++++++++++++++ scripts/start/projects/paper_healthyLscps.R | 164 ++++++++++++++ 5 files changed, 408 insertions(+), 11 deletions(-) create mode 100644 scripts/output/extra/runSEALSallocation.R create mode 100644 scripts/start/projects/paper_healthyLscps.R diff --git a/DESCRIPTION b/DESCRIPTION index 779c99c49..3a199b5db 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -21,7 +21,7 @@ Imports: m4fsdp, madrat, magclass (>= 6.14.0), - magpie4 (>= 2.11.0), + magpie4 (>= 2.12.0), MagpieNCGains, magpiesets, mip, diff --git a/config/default.cfg b/config/default.cfg index 76facad41..c651b1aeb 100644 --- a/config/default.cfg +++ b/config/default.cfg @@ -399,7 +399,7 @@ cfg$gms$kfo_rd <- "livst_rum" #def = livst_rum # * Switch for supplemental fat needed as ingredient for scp-based meat alternatives # * options: 0 (=off), 1 (=on) -cfg$gms$s15_scp_supplement_fat_meat <- 0 # def = 0 +cfg$gms$s15_scp_supplement_fat_meat <- 0 # def = 0 # * Convergence of livestock food calorie supply towards a kcal/cap/day target with or w/o substitution. # * The functional form and the start and target years of the food substitution can be specified below. @@ -695,7 +695,7 @@ cfg$gms$ageclass <- "feb21" # def = feb21 # ***--------------------- 29_cropland ----------------------------------- # * Cropland is defined as the sum of croparea, fallow land and tree cover # * Croparea is provided by 30_crop. -# * Fallow land and tree cover are defined by 29_cropland, +# * Fallow land and tree cover are defined by 29_cropland, # * (simple_apr24): Fallow land and tree cover on cropland are fixed to zero # * (detail_apr24): Fallow land and tree cover based on rules or incentives # NOTE: It is recommended to recalibrate the model when changing this setting! @@ -737,7 +737,7 @@ cfg$gms$s29_snv_scenario_target <- 2050 # def = 2050 # * plausible options: "secdforest, forestry, past, other", # * "secdforest, other", # * "secdforest, past, other" etc. -cfg$gms$land_snv <- "secdforest, forestry, past, other" #def = "secdforest, forestry, past, other" +cfg$gms$land_snv <- "secdforest, other" #def = "secdforest, other" # *** Options only available for `detail_apr24` realization *** @@ -755,7 +755,7 @@ cfg$gms$s29_treecover_map <- 0 # def = 0 # s29_treecover_target_noselect applies to all other countries. cfg$gms$s29_treecover_target <- 0 # def = 0 cfg$gms$s29_treecover_target_noselect <- 0 # def = 0 -# * Avoid loss of existing treecover (1=yes 0=no). +# * Avoid loss of existing treecover (1=yes 0=no). # * If set to 1, `s29_treecover_target` will be adjusted based existing treecover area. cfg$gms$s29_treecover_keep <- 0 # def = 0 # * Maximum share of treecover on total cropland (1) @@ -808,12 +808,12 @@ cfg$gms$policy_countries30 <- all_iso_countries ## Agroforestry settings for bioenergy trees # * Sigmoid fader for minimum area share of bioenergy trees (betr) on total cropland at cluster level # * Minimum area share of bioenergy trees on total cropland in start year -# Note: s30_betr_start applies to countries selected in policy_countries30, +# Note: s30_betr_start applies to countries selected in policy_countries30, # s30_betr_start_noselect applies to all other countries. cfg$gms$s30_betr_start <- 0 # def = 0 cfg$gms$s30_betr_start_noselect <- 0 # def = 0 # * Minimum area share of bioenergy trees on total cropland in target year -# Note: s30_betr_target applies to countries selected in policy_countries30, +# Note: s30_betr_target applies to countries selected in policy_countries30, # s30_betr_target_noselect applies to all other countries. cfg$gms$s30_betr_target <- 0 # def = 0 cfg$gms$s30_betr_target_noselect <- 0 # def = 0 @@ -841,14 +841,14 @@ cfg$gms$s30_implementation <- 1 # def = 1 # * Rotation rules: # * min (minimal constraints), default (best guess), good (good practice), # * good_20div (good practice - 20% for other crops), setaside (default plus fallow), -# * legumes (minimum share of legumes), sixfoldrotation (crops can only repeat after 6 years), +# * legumes (minimum share of legumes), sixfoldrotation (crops can only repeat after 6 years), # * agroecology (mix of previous scenarios), FSEC (similar to agroecology) # * betr0 (minimum share of short rotation agroforestry trees), betr10, betr20, betr25, betr30, betr40, betr50. cfg$gms$c30_rotation_rules <- "default" # def = "default" # * Rotation incentives: -# * none (no incentives), default (best guess), -# * legumes (increased incentives for legumes), agroecology (mix), +# * none (no incentives), default (best guess), +# * legumes (increased incentives for legumes), agroecology (mix), # * betr500 and betr1000 (incentives for short rotation agroforestry trees) cfg$gms$c30_rotation_incentives <- "none" # def = "none" @@ -2072,6 +2072,9 @@ cfg$gms$s80_secondsolve <- 0 # def = 0 # bjoernAR6_C_SSP2-NPi.mif, bjoernAR6_C_SSP2-PkBudg900.mif cfg$magicc_emis_scen <- "bjoernAR6_C_SSP2-NDC.mif" +# Years in which SEALS downscaling is applied +cfg$seals_years <- c(2020, 2050) + # Decide whether the runs should be run sequentially (TRUE), # or in parallel (FALSE) # NA means that this decision is taken automatically diff --git a/scripts/output/extra/disaggregation.R b/scripts/output/extra/disaggregation.R index 2592a29f1..e9e31f336 100644 --- a/scripts/output/extra/disaggregation.R +++ b/scripts/output/extra/disaggregation.R @@ -68,7 +68,6 @@ if (length(map_file) > 1) { base::message(message) x <- .fixCoords(x) write.magpie(x, file, comment = comment) - write.magpie(x, sub(".mz", ".nc", file), comment = comment) } .dissagcrop <- function(gdx, land_hr, map_file) { diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R new file mode 100644 index 000000000..fe6b865d9 --- /dev/null +++ b/scripts/output/extra/runSEALSallocation.R @@ -0,0 +1,231 @@ +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) +# | authors, and contributors see CITATION.cff file. This file is part +# | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of +# | AGPL-3.0, you are granted additional permissions described in the +# | MAgPIE License Exception, version 1.0 (see LICENSE file). +# | Contact: magpie@pik-potsdam.de + +# -------------------------------------------------------------- +# description: Starts SEALS allocation run based on gridded MAgPIE land cover projections (SEALS) +# comparison script: FALSE +# --------------------------------------------------------------- + +# Version 1.00 - Patrick v. Jeetze, Pascal Sauer +# 1.00: first working version + +library(gms) +library(gdx) +library(magpie4) +library(filelock) + +message("Initialising SEALS allocation run") + +############################# BASIC CONFIGURATION ####################################### +if (!exists("source_include")) { + title <- NULL + outputdir <- NULL + + # Define arguments that can be read from command line + readArgs("outputdir", "title") +} +######################################################################################### + +# ======================== +# User settings +# ======================== + +### SEALS python environment name +# see https://justinandrewjohnson.com/earth_economy_devstack/installation.html +# for instructions on how to set up a python environment for SEALS +sealsEnv <- "seals_dev" + +### Path to miniforge installation +# For instructions on how to install miniforge see https://github.com/conda-forge/miniforge +miniforgePath <- "/home/vjeetze/miniforge3/bin/activate" + +### Path to SEALS base input file directory +dirBaseFiles <- "/p/projects/magpie/users/vjeetze/seals/files" + +### Path to SEALS code base +dirSEALS <- "/p/projects/magpie/users/vjeetze/seals/files/seals/seals_dev" + + +# ======================== +# Prepare MAgPIE output +# ======================== + +message("Script started for output directory: ", outputdir) + +cfg <- gms::loadConfig(file.path(outputdir, "config.yml")) +title <- cfg$title + +if (length(cfg$seals_years) != 0) { + rep_years <- cfg$seals_years[cfg$seals_years > 2020] + rep_years <- c(2020, rep_years) +} else { + rep_years <- seq(2020, 2050, 5) +} + +# Restructure data to conform to SEALS +reportLandUseForSEALS( + magCellLand = "cell.land_0.5_share.mz", + outFile = paste0("cell.land_0.5_SEALS_", title, ".nc"), + dir = outputdir, selectyears = rep_years +) + + +# ======================== +# Setup SEALS run +# ======================== + +# -------------------------------- +# Prepare SEALS start script +# -------------------------------- + +.setupSEALSrun <- function(title, dir, dirProject, dirSEALS, dirBaseFiles) { + if (!dir.exists(file.path(dirProject, "scripts"))) { + dir.create(file.path(dirProject, "scripts"), recursive = TRUE) + } + + file.copy( + from = list.files(file.path(dirSEALS, "seals"), full.names = TRUE), + to = file.path(dirProject, "scripts"), + overwrite = TRUE, + recursive = TRUE + ) + + if (!dir.exists(file.path(dirProject, "inputs"))) { + dir.create(file.path(dirProject, "inputs"), recursive = TRUE) + } + + file.copy( + from = file.path(dir, paste0("seals_scenario_config_", title, ".csv")), + to = file.path(dirProject, "inputs", paste0("seals_scenario_config_", title, ".csv")), + overwrite = TRUE + ) + + main <- readLines(file.path(dirProject, "scripts/run_test_standard.py")) + + main[min(which(grepl(" p.user_dir =", main)))] <- paste0(" p.user_dir = \'", dirBaseFiles, "\'") + main[min(which(grepl(" p.extra_dirs", main)))] <- paste0(" p.extra_dirs = '.'") + main[min(which(grepl(" p.project_name =", main)))] <- paste0(" p.project_name = \'", title, "\'") + main[min(which(grepl(" p.project_dir =", main)))] <- paste0( + " p.project_dir = \'", normalizePath(dirProject), "\'" + ) + main[min(which(grepl("hb.pretty_time()", main)))] <- " " + main[min(which(grepl(" p.base_data_dir =", main)))] <- paste0( + " p.base_data_dir = \'", dirBaseFiles, "/base_data\'" + ) + main[min(which(grepl(" p.scenario_definitions_filename =", main)))] <- paste0( + " p.scenario_definitions_filename = \'", paste0("seals_scenario_config_", title, ".csv"), "\'" + ) + + writeLines(main, file.path(dirProject, "scripts", paste0("run_seals_", title, ".py"))) + + + if (!dir.exists(file.path(dirProject, "run_submit"))) { + dir.create(file.path(dirProject, "run_submit"), recursive = TRUE) + } +} + +# -------------------------------- +# Submit run +# -------------------------------- + +.submitSEALS <- function(title, dirProject, miniforgePath, sealsEnv, qos = "short", + slurmID = FALSE, dependsID = NULL) { + submitFile <- file.path(dirProject, "run_submit", paste0("submit_seals_", title, ".sh")) + + submit <- c( + "#!/bin/bash", "\n", + paste0("#SBATCH --qos=", qos), + ifelse(qos %in% c("priority", "standby"), "#SBATCH --partition=priority", "#SBATCH --partition=standard"), + "#SBATCH --job-name=seals_allocation", + paste0("#SBATCH --chdir=", normalizePath(file.path(dirProject, "scripts"))), + "#SBATCH --output=outfile_%j.out", + "#SBATCH --error=outfile_%j.err", + "#SBATCH --mail-type=END,FAIL", + "#SBATCH --time=3:00:00", + "#SBATCH --nodes=1", + "#SBATCH --ntasks=1", + ifelse(!is.null(dependsID), paste0( + "#SBATCH --dependency=afterok:", dependsID, + "\n#SBATCH --kill-on-invalid-dep=yes" + ), + "#SBATCH --dependency=singleton" + ), "\n", + ifelse(qos == "priority", "#SBATCH --cpus-per-task=64", "#SBATCH --cpus-per-task=128"), "\n", + paste("source", miniforgePath), + paste("conda activate", sealsEnv), "\n", + paste("python", paste0("run_seals_", title, ".py")) + ) + writeLines(submit, submitFile) + + id <- system(paste("sbatch --parsable", submitFile), intern = TRUE) + if (slurmID) { + return(id) + } +} + +# create output directory +dirProject <- "./output/seals" + +if (!dir.exists(file.path(dirProject))) { + dir.create(file.path(dirProject), recursive = TRUE) +} + +iniLock <- file.path(dirProject, ".lock") +lockOn <- filelock::lock(iniLock, exclusive = TRUE, timeout = Inf) +Sys.chmod(iniLock, mode = "0664") + +# -------------------------------- +# Run SEALS allocation +# -------------------------------- + +if (!is.null(lockOn)) { + sealsLock <- file.path(dirProject, "seals.lock") + + if (!file.exists(sealsLock) || file.size(sealsLock) == 0) { + message("Starting SEALS allocation with input data creation.") + .setupSEALSrun( + title = title, + dir = outputdir, + dirProject = dirProject, + dirSEALS = dirSEALS, + dirBaseFiles = dirBaseFiles + ) + + id <- .submitSEALS( + title = title, + dirProject = dirProject, + miniforgePath = miniforgePath, + sealsEnv = sealsEnv, + qos = "short", + slurmID = TRUE + ) + + writeLines(id, sealsLock) + } else { + message("Starting SEALS allocation using existing input data.") + id <- readLines(sealsLock) + + .setupSEALSrun( + title = title, + dir = outputdir, + dirProject = dirProject, + dirSEALS = dirSEALS, + dirBaseFiles = dirBaseFiles + ) + + .submitSEALS( + title = title, + dirProject = dirProject, + miniforgePath = miniforgePath, + sealsEnv = sealsEnv, + qos = "short", + slurmID = FALSE, + dependsID = id + ) + } + unlink(iniLock) +} diff --git a/scripts/start/projects/paper_healthyLscps.R b/scripts/start/projects/paper_healthyLscps.R new file mode 100644 index 000000000..0ea21e921 --- /dev/null +++ b/scripts/start/projects/paper_healthyLscps.R @@ -0,0 +1,164 @@ +# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | authors, and contributors see CITATION.cff file. This file is part +# | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of +# | AGPL-3.0, you are granted additional permissions described in the +# | MAgPIE License Exception, version 1.0 (see LICENSE file). +# | Contact: magpie@pik-potsdam.de + +# ---------------------------------------------------------- +# description: Healthy Landscapes Paper +# ---------------------------------------------------------- + +rev <- "rev9" + +# Note: rev4 used MAgPIE 4.6.9 +# Note: rev6 used MAgPIE 4.6.10 +# Note: rev7 used MAgPIE 4.6.11 +# Note: rev8 used MAgPIE dev 16 Sep 2024 +# Note: rev9 used MAgPIE 4.8.2 + +###################################### +#### Script to start a MAgPIE run #### +###################################### + +library(gms) +library(lucode2) +library(magclass) +library(gdx2) + +# Load start functions +source("scripts/start_functions.R") + +# ==================== +# Calibration +# ==================== + +source("config/default.cfg") + +cfg$title <- "calib_run" +cfg$output <- c("rds_report", "validation_short") +cfg$force_replace <- TRUE + +cfg$recalibrate <- TRUE +cfg$recalibrate_landconversion_cost <- TRUE + +cfg$best_calib <- TRUE +cfg$best_calib_landconversion_cost <- FALSE + +# cc is new default +cfg <- setScenario(cfg, "nocc_hist") + +# New iso food realisation +cfg$gms$food <- "anthro_iso_jun22" + +cfg$gms$s14_use_yield_calib <- 1 + +# sticky +cfg$gms$factor_costs <- "sticky_feb18" + +# marginal land scenario +cfg$gms$c30_marginal_land <- "q33_marginal" + +start_run(cfg = cfg) +calib_tgz <- magpie4::submitCalibration(paste(rev, "healthyLscps", sep = "_")) + +# ==================== +# Scenario runs +# ==================== + +prefix <- paste(rev, "healthyLscps", sep = "_") + +SSPs <- c("SSP2", "SSP1", "SSP3") + +scenarios <- c( + "REF", "DIET100", "DIET50", + "BIOS", "DIET100-BIOS", "DIET50-BIOS", + "landConservation", "DIET100-landConservation", "DIET50-landConservation", + "noNatureLoss2030", "DIET100-noNatureLoss2030", "DIET50-noNatureLoss2030", + "SNV20", "DIET100-SNV20", "DIET50-SNV20" +) + +for (ssp in SSPs) { + for (scen in scenarios) { + actions <- unlist(strsplit(scen, "-")) + + source("config/default.cfg") + + cfg$qos <- "short_maxMem" + + cfg$results_folder <- "output/:title::date:" + + cfg$output <- c( + "output_check", "extra/disaggregation", "rds_report", "extra/runSEALSallocation" + ) + + cfg$input["calibration"] <- calib_tgz + + # Climate change switched off for these runs + cfg <- setScenario(cfg, "nocc_hist") + + # New iso food realisation + cfg$gms$food <- "anthro_iso_jun22" + + cfg$gms$s14_use_yield_calib <- 1 + + # sticky + cfg$gms$factor_costs <- "sticky_feb18" + + # SNV habitat defintion + cfg$gms$land_snv <- "secdforest, other" + + cfg <- setScenario(cfg, c(ssp, "NPI")) + cfg$gms$c15_food_scenario <- ssp + + cfg$seals_years <- c(2020, 2050) + + # --- Dietary transitions --- + + # 50 % convergence towards EAT lancet diet + if ("DIET50" %in% actions) { + cfg <- setScenario(cfg, "eat_lancet_diet_v1") + cfg$gms$s15_exo_foodscen_start <- 2020 + cfg$gms$s15_exo_foodscen_target <- 2050 + cfg$gms$s15_exo_foodscen_convergence <- 0.5 + cfg$gms$s15_exo_waste <- 1 + } + + # 100% convergence towards EAT lancet diet + if ("DIET100" %in% actions) { + cfg <- setScenario(cfg, "eat_lancet_diet_v1") + cfg$gms$s15_exo_foodscen_start <- 2020 + cfg$gms$s15_exo_foodscen_target <- 2050 + cfg$gms$s15_exo_foodscen_convergence <- 1 + cfg$gms$s15_exo_waste <- 1 + } + + # --- Ecosystem stewarship actions --- + + # 30by30 land conservation + if (any(actions %in% c("BIOS", "landConservation"))) { + cfg$gms$c22_protect_scenario <- "30by30" + cfg$gms$s22_conservation_start <- 2020 + cfg$gms$s22_conservation_target <- 2030 + } + + # no net nature loss + if (any(actions %in% c("BIOS", "noNatureLoss2030"))) { + cfg$gms$c44_bii_decrease <- 0 + cfg$gms$s44_start_year <- 2030 + } + + # permanent habitats in agricultural landscapes + if (any(actions %in% c("BIOS", "SNV20"))) { + cfg$gms$s29_snv_shr <- 0.2 + cfg$gms$s29_snv_scenario_start <- 2020 + cfg$gms$s29_snv_scenario_target <- 2030 + } + + # parameters are fixed to SSP2 values until 2020 + cfg <- setScenario(cfg, "fix_2020", scenario_config = "config/projects/scenario_config_year_fix.csv") + + cfg$title <- paste0(prefix, "_", ssp, "-", scen) + start_run(cfg = cfg, codeCheck = FALSE) + } +} From 1f503af21495e9d9d4eb31f4d7a4c1c570a1381c Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Wed, 18 Sep 2024 16:01:58 +0200 Subject: [PATCH 02/12] updated changelog --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95a5a39bd..e9b093e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **config** additional data update additional_data_rev4.53.tgz - **29_cropland** added option for linear and sigmoidal faders - **scripts** output/extra/highres.R use default 13_tc realization +- **scripts** `.nc` files are no longer created by default after disaggregation ### added @@ -34,9 +35,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **cropland** added option for discarding initial treecover on cropland - **script** added output script for conversion of validation.mif file into validation.rds - **script** for downscaling to 0.25 deg using LUH2v2h as reference via mrdownscale +- **config** added `scenario_config_year_fix.csv` for choosing until when parameters are fixed to SSP2 values +- **scripts** added start script for 'Healthly Landscapes' paper `paper_healthyLscps.R` +- **scripts** added script for automatic submission of SEALS allocation runs `./extra/runSEALSallocation.R` ### removed -- + ### fixed - **scripts** fixing an error in start.R and output.R which occurred if more than one slurm job was submitted at the same time. @@ -50,7 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### changed - **29_ageclass** module 29_ageclass has been renamed to 28_ageclass to make space for `29_cropland` just before `30_croparea` - **30_crop** module `30_crop` renamed to `30_croparea`, which now only accounts for crop area. -- **30_crop** Semi-Natural Vegetation (SNV) implementation has been moved from `30_crop` to `29_cropland` +- **30_crop** Semi-Natural Vegetation (SNV) implementation has been moved from `30_crop` to `29_cropland` - **30_crop** the previous `30_crop/endo_apr21` realization has been moved to `30_croparea/simple_apr24` - **30_crop** the two realizations `penalty_apr22` and `rotation_apr22` have been merged into a single `30_croparea/detail_apr24` realization - **default.cfg** update additional data to rev4.51 From 40a442ab2590e9f427757830cf72102f742c09aa Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Wed, 18 Sep 2024 16:13:45 +0200 Subject: [PATCH 03/12] added scenario config to set SSP2 fix --- config/projects/scenario_config_year_fix.csv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 config/projects/scenario_config_year_fix.csv diff --git a/config/projects/scenario_config_year_fix.csv b/config/projects/scenario_config_year_fix.csv new file mode 100644 index 000000000..282d61934 --- /dev/null +++ b/config/projects/scenario_config_year_fix.csv @@ -0,0 +1,16 @@ +;fix_2020;fix_2025 +gms$sm_fix_SSP2;2020;2025 +gms$sm_fix_cc;2020;2025 +gms$s15_food_substitution_start;2020;2025 +gms$s15_exo_foodscen_start;2020;2025 +gms$s21_trade_tariff_startyear;2020;2025 +gms$s22_conservation_start;2020;2025 +gms$s29_snv_scenario_start;2020;2025 +gms$s29_treecover_scenario_start;2020;2025 +gms$s29_fallow_scenario_start;2020;2025 +gms$s30_betr_scenario_start;2020;2025 +gms$s30_rotation_scenario_start;2020;2025 +gms$s38_startyear_labor_substitution;2020;2025 +gms$s42_efp_startyear;2020;2025 +gms$s44_start_year;2020;2025 +gms$s70_feed_substitution_start;2020;2025 From 98778a49a5cd7e93859ed1f0bfc3869c546fdb47 Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Wed, 18 Sep 2024 17:03:06 +0200 Subject: [PATCH 04/12] updated copyright year --- scripts/start/projects/GENIE_0_default.R | 2 +- scripts/start/projects/GENIE_1_tau-export.R | 4 ++-- scripts/start/projects/GENIE_2_bm-priced.R | 6 +++--- scripts/start/projects/GENIE_3_bm-demand.R | 4 ++-- scripts/start/projects/GENIE_4_feedback.R | 10 +++++----- scripts/start/projects/paper_healthyLscps.R | 2 +- scripts/start/projects/project_EAT2p0_DeepDive.R | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/start/projects/GENIE_0_default.R b/scripts/start/projects/GENIE_0_default.R index 248e67bfa..74fde2c3c 100644 --- a/scripts/start/projects/GENIE_0_default.R +++ b/scripts/start/projects/GENIE_0_default.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the diff --git a/scripts/start/projects/GENIE_1_tau-export.R b/scripts/start/projects/GENIE_1_tau-export.R index 4859e7855..561266c5c 100644 --- a/scripts/start/projects/GENIE_1_tau-export.R +++ b/scripts/start/projects/GENIE_1_tau-export.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the @@ -52,7 +52,7 @@ cfg$info$flag <- identifierFlag cfg$results_folder <- paste0("output/", identifierFlag, "/:title:") cfg$gms$c44_bii_decrease <- 0 -cfg$gms$c22_protect_scenario <- "BH" +cfg$gms$c22_protect_scenario <- "BH" cfg$gms$s60_2ndgen_bioenergy_dem_min_post_fix <- 0 cfg$gms$c60_bioenergy_subsidy <- 0 diff --git a/scripts/start/projects/GENIE_2_bm-priced.R b/scripts/start/projects/GENIE_2_bm-priced.R index e58c1b485..5cac56463 100644 --- a/scripts/start/projects/GENIE_2_bm-priced.R +++ b/scripts/start/projects/GENIE_2_bm-priced.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the @@ -52,7 +52,7 @@ cfg$info$flag <- identifierFlag cfg$results_folder <- paste0("output/", identifierFlag, "/:title:") -### BE +### BE cfg$gms$s60_2ndgen_bioenergy_dem_min_post_fix <- 0 cfg$gms$c60_bioenergy_subsidy <- 0 beV <- c(0, 5, 7, 10, 15, 25, 45) @@ -83,7 +83,7 @@ for (bl in blV) { preflag <- paste0("MP", str_pad(mp, 2, pad = "0"), "BI", str_pad(bl * 100, 2, pad = "0")) cfg$results_folder <- paste("output", identifierFlag, preflag, ":title:", sep = "/") cfg$info$flag2 <- preflag - + cfg$gms$s15_rumdairy_scp_substitution <- mp / 100 for (be in beV) { diff --git a/scripts/start/projects/GENIE_3_bm-demand.R b/scripts/start/projects/GENIE_3_bm-demand.R index 5bca4b022..715ecb4e8 100644 --- a/scripts/start/projects/GENIE_3_bm-demand.R +++ b/scripts/start/projects/GENIE_3_bm-demand.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the @@ -51,7 +51,7 @@ cfg$info$flag <- identifierFlag cfg$results_folder <- paste0("output/", identifierFlag, "/:title:") -### BE +### BE cfg$gms$s60_2ndgen_bioenergy_dem_min_post_fix <- 0 cfg$gms$c60_bioenergy_subsidy <- 0 beV <- c(0, 5, 7, 10, 15, 25, 45) diff --git a/scripts/start/projects/GENIE_4_feedback.R b/scripts/start/projects/GENIE_4_feedback.R index 1fa097a56..2f2ced23b 100644 --- a/scripts/start/projects/GENIE_4_feedback.R +++ b/scripts/start/projects/GENIE_4_feedback.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the @@ -50,12 +50,12 @@ cfg$info$flag <- identifierFlag cfg$results_folder <- paste0("output/", identifierFlag, "/:title:") -### BE +### BE cfg$gms$s60_2ndgen_bioenergy_dem_min_post_fix <- 0 cfg$gms$c60_bioenergy_subsidy <- 0 ### Tau / Yield -cfg$gms$tc <- "exo" +cfg$gms$tc <- "exo" ### Biodiv blV <- c(0) #BII lower bound (0-1), default 0 @@ -87,7 +87,7 @@ for (bl in blV) { # "output", identifierFlag, preflag, ":title:", sep = "/" "output", identifierFlag, ":title:", sep = "/" ) - + n <- "Feedback_step13_400f" m <- n cfg$gms$c60_2ndgen_biodem <- n @@ -96,7 +96,7 @@ for (bl in blV) { ############################################## #runflag <- paste("feedback", f_flag, sep = "_") - runflag <- paste0(m, "") + runflag <- paste0(m, "") cfg$title <- paste0(preflag, runflag) diff --git a/scripts/start/projects/paper_healthyLscps.R b/scripts/start/projects/paper_healthyLscps.R index 0ea21e921..d1d3e39eb 100644 --- a/scripts/start/projects/paper_healthyLscps.R +++ b/scripts/start/projects/paper_healthyLscps.R @@ -1,4 +1,4 @@ -# | (C) 2008-2021 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the diff --git a/scripts/start/projects/project_EAT2p0_DeepDive.R b/scripts/start/projects/project_EAT2p0_DeepDive.R index 3dd9e4bf2..40fbb9d8a 100644 --- a/scripts/start/projects/project_EAT2p0_DeepDive.R +++ b/scripts/start/projects/project_EAT2p0_DeepDive.R @@ -1,4 +1,4 @@ -# | (C) 2008-2023 Potsdam Institute for Climate Impact Research (PIK) +# | (C) 2008-2024 Potsdam Institute for Climate Impact Research (PIK) # | authors, and contributors see CITATION.cff file. This file is part # | of MAgPIE and licensed under AGPL-3.0-or-later. Under Section 7 of # | AGPL-3.0, you are granted additional permissions described in the @@ -240,7 +240,7 @@ cfg <- setScenario(cfg, c("EL2_default"), scenario_config = "config/projects/sce cfg <- bau(cfg = cfg) # Mitigation cfg <- miti(cfg = cfg) -# ecoSysProtAll_agMgmtOff: +# ecoSysProtAll_agMgmtOff: # (Above ground CO2 emis from LUC in forest, forestry, natveg; All types of emis from peatland; No further CH4/N2O/other emis related to ag. management) cfg$gms$c56_emis_policy <- "ecoSysProtAll_agMgmtOff" start_run(cfg, codeCheck = FALSE) From eaadd8c395648d26adde064d34055e3af1a744bc Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Thu, 19 Sep 2024 10:49:38 +0200 Subject: [PATCH 05/12] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9b093e0e..ac433147c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **29_cropland** added option for linear and sigmoidal faders - **scripts** output/extra/highres.R use default 13_tc realization - **scripts** `.nc` files are no longer created by default after disaggregation +- **config** `cfg$gms$land_snv`changed from "secdforest, forestry, past, other" to "secdforest, other" ### added From 6aef8ede403b36302ceba22e06c49c310698860a Mon Sep 17 00:00:00 2001 From: pvjeetze <50408549+pvjeetze@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:38:52 +0200 Subject: [PATCH 06/12] Update scripts/output/extra/runSEALSallocation.R Co-authored-by: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> --- scripts/output/extra/runSEALSallocation.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index fe6b865d9..fa8184ad4 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -6,7 +6,7 @@ # | Contact: magpie@pik-potsdam.de # -------------------------------------------------------------- -# description: Starts SEALS allocation run based on gridded MAgPIE land cover projections (SEALS) +# description: Starts SEALS allocation run based on gridded MAgPIE land cover projections # comparison script: FALSE # --------------------------------------------------------------- From e859db92c918138d9202f7a23b955f78f1e2a2c3 Mon Sep 17 00:00:00 2001 From: pvjeetze <50408549+pvjeetze@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:46:09 +0200 Subject: [PATCH 07/12] Update scripts/output/extra/runSEALSallocation.R Co-authored-by: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> --- scripts/output/extra/runSEALSallocation.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index fa8184ad4..07ebf324a 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -104,7 +104,7 @@ reportLandUseForSEALS( overwrite = TRUE ) - main <- readLines(file.path(dirProject, "scripts/run_test_standard.py")) + main <- readLines(file.path(dirProject, "scripts", "run_test_standard.py")) main[min(which(grepl(" p.user_dir =", main)))] <- paste0(" p.user_dir = \'", dirBaseFiles, "\'") main[min(which(grepl(" p.extra_dirs", main)))] <- paste0(" p.extra_dirs = '.'") From b13da8f7ad53d52cf799f17023a7bdeb69416b6a Mon Sep 17 00:00:00 2001 From: pvjeetze <50408549+pvjeetze@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:47:48 +0200 Subject: [PATCH 08/12] Update scripts/output/extra/runSEALSallocation.R Co-authored-by: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> --- scripts/output/extra/runSEALSallocation.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index 07ebf324a..8f71ad912 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -139,7 +139,7 @@ reportLandUseForSEALS( submit <- c( "#!/bin/bash", "\n", paste0("#SBATCH --qos=", qos), - ifelse(qos %in% c("priority", "standby"), "#SBATCH --partition=priority", "#SBATCH --partition=standard"), + if (qos %in% c("priority", "standby")) "#SBATCH --partition=priority" else "#SBATCH --partition=standard", "#SBATCH --job-name=seals_allocation", paste0("#SBATCH --chdir=", normalizePath(file.path(dirProject, "scripts"))), "#SBATCH --output=outfile_%j.out", From c04f45ba666708497d0341c4f8824840c2cae240 Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Thu, 19 Sep 2024 14:15:02 +0200 Subject: [PATCH 09/12] changed source path of miniforge3 --- scripts/output/extra/runSEALSallocation.R | 45 +++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index fe6b865d9..93914f2bc 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -39,10 +39,6 @@ if (!exists("source_include")) { # for instructions on how to set up a python environment for SEALS sealsEnv <- "seals_dev" -### Path to miniforge installation -# For instructions on how to install miniforge see https://github.com/conda-forge/miniforge -miniforgePath <- "/home/vjeetze/miniforge3/bin/activate" - ### Path to SEALS base input file directory dirBaseFiles <- "/p/projects/magpie/users/vjeetze/seals/files" @@ -78,6 +74,36 @@ reportLandUseForSEALS( # Setup SEALS run # ======================== +# Check whether base data and local SEALS code repo exist +if (length(list.files(dirBaseFiles)) == 0) { + stop(paste( + "Please set the path to the SEALS base data", + "directory under 'User settings' of the", + "extra/runSEALSallocation.R script." + )) +} else if (length(list.files(dirBaseFiles)) == 0) { + stop(paste( + "Please set the path to your local clone", + "of the SEALS code under 'User settings' of the", + "extra/runSEALSallocation.R script." + )) +} + +### Path to miniforge installation +miniforgePath <- "/p/projects/rd3mod/miniforge3/bin/activate" + +# create output directory +dirProject <- "./output/seals" + +if (!dir.exists(file.path(dirProject))) { + dir.create(file.path(dirProject), recursive = TRUE) +} + +iniLock <- file.path(dirProject, ".lock") +lockOn <- filelock::lock(iniLock, exclusive = TRUE, timeout = Inf) +Sys.chmod(iniLock, mode = "0664") + + # -------------------------------- # Prepare SEALS start script # -------------------------------- @@ -167,17 +193,6 @@ reportLandUseForSEALS( } } -# create output directory -dirProject <- "./output/seals" - -if (!dir.exists(file.path(dirProject))) { - dir.create(file.path(dirProject), recursive = TRUE) -} - -iniLock <- file.path(dirProject, ".lock") -lockOn <- filelock::lock(iniLock, exclusive = TRUE, timeout = Inf) -Sys.chmod(iniLock, mode = "0664") - # -------------------------------- # Run SEALS allocation # -------------------------------- From 5e8f173589a7db42c99e294cbd6e8cf3438ca740 Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Thu, 19 Sep 2024 14:18:03 +0200 Subject: [PATCH 10/12] bugfix --- scripts/output/extra/runSEALSallocation.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index 5823fcda4..0b9f2eafa 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -81,7 +81,7 @@ if (length(list.files(dirBaseFiles)) == 0) { "directory under 'User settings' of the", "extra/runSEALSallocation.R script." )) -} else if (length(list.files(dirBaseFiles)) == 0) { +} else if (length(list.files(dirSEALS)) == 0) { stop(paste( "Please set the path to your local clone", "of the SEALS code under 'User settings' of the", From ff6244b0a20a168e16e616c185e489dd8c66cf22 Mon Sep 17 00:00:00 2001 From: pvjeetze <50408549+pvjeetze@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:58:05 +0200 Subject: [PATCH 11/12] Update scripts/output/extra/runSEALSallocation.R Co-authored-by: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> --- scripts/output/extra/runSEALSallocation.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index 0b9f2eafa..21b289f02 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -94,6 +94,8 @@ miniforgePath <- "/p/projects/rd3mod/miniforge3/bin/activate" # create output directory dirProject <- "./output/seals" +message(paste("Stitched SEALS allocation outputs are written to", + "'./output/seals/intermediate/stitched_lulc_simplified_scenarios'")) if (!dir.exists(file.path(dirProject))) { dir.create(file.path(dirProject), recursive = TRUE) From 759ff44df1f4cdc64b489731b7133a543f98ab8b Mon Sep 17 00:00:00 2001 From: pvjeetze Date: Fri, 20 Sep 2024 12:03:15 +0200 Subject: [PATCH 12/12] additional fixes, including run submissions of Foote --- CHANGELOG.md | 1 + config/default.cfg | 20 +++++++++---------- scripts/output/extra/highres.R | 4 ++-- scripts/output/extra/runSEALSallocation.R | 15 ++++++++++---- scripts/projects/fsec.R | 2 +- scripts/run_submit/submit_priority.sh | 1 + ...y_maxMem.sh => submit_priority_highMem.sh} | 2 +- scripts/run_submit/submit_short.sh | 1 + ...hort_maxMem.sh => submit_short_highMem.sh} | 2 +- scripts/run_submit/submit_standby.sh | 1 + scripts/run_submit/submit_standby_dayMax.sh | 1 + ...by_maxMem.sh => submit_standby_highMem.sh} | 2 +- ...ax.sh => submit_standby_highMem_dayMax.sh} | 2 +- scripts/slurmOutput.yml | 8 ++++---- scripts/start/projects/paper_healthyLscps.R | 2 +- scripts/start/projects/paper_peatlandTax.R | 20 +++++++++---------- scripts/start/projects/project_ABCDR.R | 2 +- scripts/start/projects/project_EAT2p0.R | 2 +- .../start/projects/project_EAT2p0_DeepDive.R | 2 +- .../start/projects/project_LAMACLIMA_WP4.R | 2 +- 20 files changed, 52 insertions(+), 40 deletions(-) rename scripts/run_submit/{submit_priority_maxMem.sh => submit_priority_highMem.sh} (86%) rename scripts/run_submit/{submit_short_maxMem.sh => submit_short_highMem.sh} (86%) rename scripts/run_submit/{submit_standby_maxMem.sh => submit_standby_highMem.sh} (86%) rename scripts/run_submit/{submit_standby_maxMem_dayMax.sh => submit_standby_highMem_dayMax.sh} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac433147c..7fbdc8f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **scripts** output/extra/highres.R use default 13_tc realization - **scripts** `.nc` files are no longer created by default after disaggregation - **config** `cfg$gms$land_snv`changed from "secdforest, forestry, past, other" to "secdforest, other" +- **config** Submissions with high memory requirements now get 16 CPUs and 80GB of memory to account for the new specifications of PIK's new HPC 'Foote'. `maxMem` was renamed to `highMem`, because 80GB of memory does not correspond to the maximum available memory of a compute node. ### added diff --git a/config/default.cfg b/config/default.cfg index c651b1aeb..57b6d0970 100644 --- a/config/default.cfg +++ b/config/default.cfg @@ -26,7 +26,7 @@ cfg$input <- c(regional = "rev4.111_h12_magpie.tgz", cellular = "rev4.111_h12_fd712c0b_cellularmagpie_c200_MRI-ESM2-0-ssp370_lpjml-8e6c5eb1.tgz", validation = "rev4.111_h12_validation.tgz", additional = "additional_data_rev4.53.tgz", - calibration = "calibration_H12_26Mar24.tgz") + calibration = "calibration_H12_26Mar24_fixed.tgz") # NOTE: It is recommended to recalibrate the model when changing cellular input data # as well as for any other setting that would affect initial values in the model, @@ -2083,15 +2083,15 @@ cfg$sequential <- NA # Selection of QOS to be used for submitted runs on cluster. # Will be ignored for all other runs. -# * options: short (24h max, no preemption) -# * short_maxMem (same as short but with 16 CPUs and max Memory) -# * medium (1 week max, no preemption) -# * priority (immediate start, but slots limited to 5 in parallel) -# * priority_maxMem (same as priority but with 16 CPUs and max Memory) -# * standby (1 week max, preemption possible) -# * standby_dayMax (24h max, preemption possible) -# * standby_maxMem_dayMax (same as standby_dayMax but with 16 CPUs and max Memory) -# * standby_maxMem (same as standby but with 16 CPUs and max Memory) +# * options: short (24h max, 3 CPUs, no preemption) +# * short_highMem (same as short but with 16 CPUs and 80GB of memory) +# * medium (1 week max, 3 CPUs, no preemption) +# * priority (immediate start, 3 CPUs, but slots limited to 5 in parallel) +# * priority_highMem (same as priority but with 16 CPUs and 80GB of memory) +# * standby (1 week max, 3 CPUs, preemption possible) +# * standby_dayMax (24h max, 3 CPUs, preemption possible) +# * standby_highMem_dayMax (same as standby_dayMax but with 16 CPUs and 80GB of memory) +# * standby_highMem (same as standby but with 16 CPUs and 80GB of memory) # * NULL (educated guess of best option based # * available resources) cfg$qos <- NULL # def = NULL diff --git a/scripts/output/extra/highres.R b/scripts/output/extra/highres.R index 3d93842c7..c1768e03d 100644 --- a/scripts/output/extra/highres.R +++ b/scripts/output/extra/highres.R @@ -104,7 +104,7 @@ highres <- function(cfg = cfg, res = "c1000", tc = NULL) { cfg$gms$s_use_gdx <- 2 #max resources for parallel runs - cfg$qos <- "standby_maxMem_dayMax" + cfg$qos <- "standby_highMem_dayMax" # set force download to FALSE # otherwise data is download again when calling start_run(), which overwrites @@ -156,7 +156,7 @@ highres <- function(cfg = cfg, res = "c1000", tc = NULL) { write.magpie(readGDX(gdx,"f56_pollutant_prices_emulator"),"modules/56_ghg_policy/input/f56_pollutant_prices_emulator.cs3") write.magpie(readGDX(gdx,"f60_bioenergy_dem_coupling"),"modules/60_bioenergy/input/reg.2ndgen_bioenergy_demand.csv") write.magpie(readGDX(gdx,"f60_bioenergy_dem_emulator"),"modules/60_bioenergy/input/glo.2ndgen_bioenergy_demand.csv") - + #get regional afforestation patterns from low resolution run with c200 aff <- dimSums(landForestry(gdx)[,,c("aff","ndc")],dim=3) #Take away initial NDC area for consistency with global afforestation limit diff --git a/scripts/output/extra/runSEALSallocation.R b/scripts/output/extra/runSEALSallocation.R index 21b289f02..26b8fcfdf 100644 --- a/scripts/output/extra/runSEALSallocation.R +++ b/scripts/output/extra/runSEALSallocation.R @@ -94,8 +94,6 @@ miniforgePath <- "/p/projects/rd3mod/miniforge3/bin/activate" # create output directory dirProject <- "./output/seals" -message(paste("Stitched SEALS allocation outputs are written to", - "'./output/seals/intermediate/stitched_lulc_simplified_scenarios'")) if (!dir.exists(file.path(dirProject))) { dir.create(file.path(dirProject), recursive = TRUE) @@ -203,7 +201,11 @@ if (!is.null(lockOn)) { sealsLock <- file.path(dirProject, "seals.lock") if (!file.exists(sealsLock) || file.size(sealsLock) == 0) { - message("Starting SEALS allocation with input data creation.") + message(paste( + "Starting SEALS allocation with input data creation.\n", + "Stitched SEALS allocation outputs will be written to", + "'./output/seals/intermediate/stitched_lulc_simplified_scenarios'" + )) .setupSEALSrun( title = title, dir = outputdir, @@ -223,7 +225,12 @@ if (!is.null(lockOn)) { writeLines(id, sealsLock) } else { - message("Starting SEALS allocation using existing input data.") + message(paste( + "Starting SEALS allocation using existing input data.\n", + "Stitched SEALS allocation outputs will be written to", + "'./output/seals/intermediate/stitched_lulc_simplified_scenarios'" + )) + id <- readLines(sealsLock) .setupSEALSrun( diff --git a/scripts/projects/fsec.R b/scripts/projects/fsec.R index 0449a6028..f7f3302ba 100644 --- a/scripts/projects/fsec.R +++ b/scripts/projects/fsec.R @@ -214,7 +214,7 @@ fsecScenario <- function(scenario) { # general cfg$title <- paste(v, scenario, sep = "") cfg$recalibrate <- FALSE - cfg$qos <- "standby_maxMem_dayMax" + cfg$qos <- "standby_highMem_dayMax" cfg$output <- c("extra/highres", "extra/disaggregation", "projects/FSEC_nitrogenPollution", diff --git a/scripts/run_submit/submit_priority.sh b/scripts/run_submit/submit_priority.sh index 33c8496d5..b23c5bd67 100644 --- a/scripts/run_submit/submit_priority.sh +++ b/scripts/run_submit/submit_priority.sh @@ -5,5 +5,6 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=3 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_priority_maxMem.sh b/scripts/run_submit/submit_priority_highMem.sh similarity index 86% rename from scripts/run_submit/submit_priority_maxMem.sh rename to scripts/run_submit/submit_priority_highMem.sh index ae58b9695..feae97160 100644 --- a/scripts/run_submit/submit_priority_maxMem.sh +++ b/scripts/run_submit/submit_priority_highMem.sh @@ -5,6 +5,6 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=16 -#SBATCH --mem-per-cpu=0 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_short.sh b/scripts/run_submit/submit_short.sh index 4e7b6edfc..433ed3308 100644 --- a/scripts/run_submit/submit_short.sh +++ b/scripts/run_submit/submit_short.sh @@ -5,5 +5,6 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=3 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_short_maxMem.sh b/scripts/run_submit/submit_short_highMem.sh similarity index 86% rename from scripts/run_submit/submit_short_maxMem.sh rename to scripts/run_submit/submit_short_highMem.sh index 2016794e7..cf8f9669a 100644 --- a/scripts/run_submit/submit_short_maxMem.sh +++ b/scripts/run_submit/submit_short_highMem.sh @@ -5,7 +5,7 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=16 -#SBATCH --mem-per-cpu=0 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_standby.sh b/scripts/run_submit/submit_standby.sh index f0bff3233..e3ab2dd5b 100644 --- a/scripts/run_submit/submit_standby.sh +++ b/scripts/run_submit/submit_standby.sh @@ -5,5 +5,6 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=3 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_standby_dayMax.sh b/scripts/run_submit/submit_standby_dayMax.sh index d7836390d..aa6b3673d 100644 --- a/scripts/run_submit/submit_standby_dayMax.sh +++ b/scripts/run_submit/submit_standby_dayMax.sh @@ -5,6 +5,7 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=3 +#SBATCH --mem-per-cpu=5G #SBATCH --time=24:00:00 Rscript submit.R diff --git a/scripts/run_submit/submit_standby_maxMem.sh b/scripts/run_submit/submit_standby_highMem.sh similarity index 86% rename from scripts/run_submit/submit_standby_maxMem.sh rename to scripts/run_submit/submit_standby_highMem.sh index 23b099943..4791ed844 100644 --- a/scripts/run_submit/submit_standby_maxMem.sh +++ b/scripts/run_submit/submit_standby_highMem.sh @@ -5,6 +5,6 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=16 -#SBATCH --mem-per-cpu=0 +#SBATCH --mem-per-cpu=5G Rscript submit.R diff --git a/scripts/run_submit/submit_standby_maxMem_dayMax.sh b/scripts/run_submit/submit_standby_highMem_dayMax.sh similarity index 88% rename from scripts/run_submit/submit_standby_maxMem_dayMax.sh rename to scripts/run_submit/submit_standby_highMem_dayMax.sh index ac3ba4d30..674bc10f5 100644 --- a/scripts/run_submit/submit_standby_maxMem_dayMax.sh +++ b/scripts/run_submit/submit_standby_highMem_dayMax.sh @@ -5,7 +5,7 @@ #SBATCH --output=slurm.log #SBATCH --mail-type=END,FAIL #SBATCH --cpus-per-task=16 -#SBATCH --mem-per-cpu=0 +#SBATCH --mem-per-cpu=5G #SBATCH --time=24:00:00 Rscript submit.R diff --git a/scripts/slurmOutput.yml b/scripts/slurmOutput.yml index f493010be..d3e35abe5 100644 --- a/scripts/slurmOutput.yml +++ b/scripts/slurmOutput.yml @@ -1,5 +1,5 @@ slurmjobs: - SLURM standby: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --mem-per-cpu=8000 --wrap=\"Rscript %SCRIPT\" --qos=standby --time=24:00:00" - SLURM standby maxMem: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --mem-per-cpu=8000 --wrap=\"Rscript %SCRIPT\" --qos=standby --time=24:00:00 --mem-per-cpu=0 --cpus-per-task=16" - SLURM priority: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --mem-per-cpu=8000 --wrap=\"Rscript %SCRIPT\" --qos=priority" - SLURM priority maxMem: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --mem-per-cpu=8000 --wrap=\"Rscript %SCRIPT\" --qos=priority --mem-per-cpu=0 --cpus-per-task=16" + SLURM standby: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --wrap=\"Rscript %SCRIPT\" --qos=standby --mem-per-cpu=5G --cpus-per-task=3" + SLURM standby highMem: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --wrap=\"Rscript %SCRIPT\" --qos=standby --time=24:00:00 --mem-per-cpu=5G --cpus-per-task=16" + SLURM priority: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --time=200 --wrap=\"Rscript %SCRIPT\" --qos=priority --mem-per-cpu=5G --cpus-per-task=3" + SLURM priority highMem: "sbatch --job-name=scripts-output --output=logs/out-%NAME-%j.out --error=logs/out-%NAME-%j.err --mail-type=END,FAIL --wrap=\"Rscript %SCRIPT\" --qos=priority --time=24:00:00 --mem-per-cpu=5G --cpus-per-task=16" diff --git a/scripts/start/projects/paper_healthyLscps.R b/scripts/start/projects/paper_healthyLscps.R index d1d3e39eb..9e5ff9776 100644 --- a/scripts/start/projects/paper_healthyLscps.R +++ b/scripts/start/projects/paper_healthyLscps.R @@ -84,7 +84,7 @@ for (ssp in SSPs) { source("config/default.cfg") - cfg$qos <- "short_maxMem" + cfg$qos <- "short_highMem" cfg$results_folder <- "output/:title::date:" diff --git a/scripts/start/projects/paper_peatlandTax.R b/scripts/start/projects/paper_peatlandTax.R index 2ed3cdbbe..91824915f 100644 --- a/scripts/start/projects/paper_peatlandTax.R +++ b/scripts/start/projects/paper_peatlandTax.R @@ -30,7 +30,7 @@ cfg$results_folder_highres <- "output" cfg$output <- c(cfg$output, "extra/highres") cfg$force_replace <- TRUE cfg$force_download <- FALSE -cfg$qos <- "standby_maxMem_dayMax" +cfg$qos <- "standby_highMem_dayMax" # support function to create standardized title .title <- function(cfg, ...) @@ -55,7 +55,7 @@ calc_ghgprice <- function() { T0 <- read.magpie("modules/56_ghg_policy/input/f56_pollutant_prices.cs3") T0 <- collapseNames(T0[, , getNames(T0, dim = 2)[1]]) T0[, , ] <- 0 - + #T200 200 USD/tCO2 in 2050 T200 <- new.magpie(getRegions(T0), c(seq(1995, 2025, by = 5), 2050, 2100, 2150), getNames(T0), fill = 0) T200[, "y2025", "co2_c"] <- 0 @@ -68,12 +68,12 @@ calc_ghgprice <- function() { T200[, , "n2o_n_direct"] <- T200[, , "co2_c"] * 265 * 44 / 28 T200[, , "n2o_n_indirect"] <- T200[, , "co2_c"] * 265 * 44 / 28 T200[, , "co2_c"] <- T200[, , "co2_c"] * 44 / 12 - + T50 <- T200 * 0.25 T100 <- T200 * 0.5 T400 <- T200 * 2 T800 <- T200 * 4 - + GHG <- mbind( add_dimension(T0, dim = 3.2, add = "scen", nm = "T0-GHG"), add_dimension( @@ -107,16 +107,16 @@ calc_ghgprice <- function() { nm = "T800-GHG" ) ) - + CO2 <- GHG CO2[, , c("ch4", "n2o_n_direct", "n2o_n_indirect")] <- 0 getNames(CO2, dim = 2) <- gsub("GHG", "CO2", getNames(CO2, dim = 2)) - + GHGCH4GWP20 <- GHG GHGCH4GWP20[, , "ch4"] <- GHGCH4GWP20[, , "ch4"] / 28 * 84 getNames(GHGCH4GWP20, dim = 2) <- gsub("GHG", "GHG-GWP20", getNames(GHGCH4GWP20, dim = 2)) - + GHG <- mbind(CO2, GHG, GHGCH4GWP20) if (!dir.exists("./patch_inputdata")) dir.create("./patch_inputdata") @@ -126,7 +126,7 @@ calc_ghgprice <- function() { write.magpie(GHG, file_name = "patch_inputdata/patchGHGprices/f56_pollutant_prices.cs3") tardir("patch_inputdata/patchGHGprices", "patch_inputdata/patchGHGprices.tgz") - + unlink("patch_inputdata/patchGHGprices", recursive = TRUE) return(getNames(GHG, dim = 2)) } @@ -146,7 +146,7 @@ cfg$gms$s56_c_price_induced_aff <- 0 ## Start scenarios for (res in c("c400")) { - if (res == "c400") + if (res == "c400") cfg$input['cellular'] <- "rev4.111_36f73207_44a213b6_cellularmagpie_c400_MRI-ESM2-0-ssp370_lpjml-8e6c5eb1_clusterweight-ba4466a8.tgz" else if (res == "c1000") { cfg$input['cellular'] <- "rev4.111_36f73207_10f98ac1_cellularmagpie_c1000_MRI-ESM2-0-ssp370_lpjml-8e6c5eb1_clusterweight-ba4466a8.tgz" @@ -156,7 +156,7 @@ for (res in c("c400")) { cfg$gms$c56_mute_ghgprices_until <- "y2150" cfg$gms$c56_pollutant_prices <- "T0-CO2" start_run(cfg, codeCheck = FALSE) - + ## Policy scenarios for (tax in c("T50-CO2", "T100-CO2", diff --git a/scripts/start/projects/project_ABCDR.R b/scripts/start/projects/project_ABCDR.R index 981fb823e..8b1480c53 100644 --- a/scripts/start/projects/project_ABCDR.R +++ b/scripts/start/projects/project_ABCDR.R @@ -25,7 +25,7 @@ cfg$results_folder <- "output/:title:" cfg$force_replace <- TRUE cfg$force_download <- FALSE -cfg$qos <- "standby_maxMem_dayMax" +cfg$qos <- "standby_highMem_dayMax" # support function to create standardized title .title <- function(cfg, ...) return(paste(cfg$info$flag, sep="_",...)) diff --git a/scripts/start/projects/project_EAT2p0.R b/scripts/start/projects/project_EAT2p0.R index 9b1c073d2..83c759c07 100644 --- a/scripts/start/projects/project_EAT2p0.R +++ b/scripts/start/projects/project_EAT2p0.R @@ -27,7 +27,7 @@ cfg$results_folder <- "output/:title::date:" cfg$force_download <- TRUE # Special outputs required for Deep Dive -cfg$qos <- "standby_maxMem_dayMax" +cfg$qos <- "standby_highMem_dayMax" cfg$output <- c( "output_check", # "extra/highres", # do manually on last magpie run diff --git a/scripts/start/projects/project_EAT2p0_DeepDive.R b/scripts/start/projects/project_EAT2p0_DeepDive.R index 40fbb9d8a..704e05d09 100644 --- a/scripts/start/projects/project_EAT2p0_DeepDive.R +++ b/scripts/start/projects/project_EAT2p0_DeepDive.R @@ -27,7 +27,7 @@ cfg$results_folder <- "output/:title::date:" cfg$force_download <- TRUE # Special outputs required for Deep Dive -cfg$qos <- "standby_maxMem_dayMax" +cfg$qos <- "standby_highMem_dayMax" cfg$output <- c( "output_check", "extra/highres", diff --git a/scripts/start/projects/project_LAMACLIMA_WP4.R b/scripts/start/projects/project_LAMACLIMA_WP4.R index 709d58052..a8b1ac6f1 100644 --- a/scripts/start/projects/project_LAMACLIMA_WP4.R +++ b/scripts/start/projects/project_LAMACLIMA_WP4.R @@ -39,7 +39,7 @@ cfg$gms$c37_labor_uncertainty <- "ensmean" cfg$results_folder <- "output/:title:" cfg$output <- c("rds_report","extra/disaggregation","extra/disaggregation_LUH2","extra/highres") -cfg$qos <- "priority_maxMem" +cfg$qos <- "priority_highMem" #### Main scenarios