repair reportingREMIND2MAgPIE

scripts/output/export/xlsx_IIASA.R

@@ -67,7 +67,7 @@ lucode2::readArgs("outputdirs", "filename_prefix", "outputFilename", "model",
 if (is.null(mapping)) {
   mapping <- gms::chooseFromList(names(piamInterfaces::templateNames()), type = "mapping template")
 }
-if (length(mapping) == 0 || ! all(file.exists(mapping) | mapping %in% names(templateNames())))
+if (length(mapping) == 0 || ! all(file.exists(mapping) | mapping %in% names(templateNames()))) {
   stop("mapping='", paste(mapping, collapse = ", "), "' not found.")
 }
 if (exists("iiasatemplate") && ! is.null(iiasatemplate) && ! file.exists(iiasatemplate)) {

scripts/output/single/reporting.R

@@ -50,11 +50,8 @@ load(configfile, envir = envir)
 
 # produce REMIND reporting *.mif based on gdx information
 message("\n### start generation of mif files at ", Sys.time())
-tmp <- try(convGDX2MIF(gdx, gdx_refpolicycost = gdx_refpolicycost, file = remind_reporting_file,
-                       scenario = scenario, gdx_ref = gdx_ref)) # try to execute convGDX2MIF
-if (inherits(tmp, "try-error")) {
-  convGDX2MIF_REMIND2MAgPIE(gdx, file = remind_reporting_file, scenario = scenario)
-}
+convGDX2MIF(gdx, gdx_refpolicycost = gdx_refpolicycost, file = remind_reporting_file,
+            scenario = scenario, gdx_ref = gdx_ref)
 
 #  MAGICC code not working with REMIND-EU
 # generate MAGICC reporting and append to REMIND reporting
@@ -122,11 +119,9 @@ if (! is.null(magpie_reporting_file) && file.exists(magpie_reporting_file)) {
 message("### end generation of mif files at ", Sys.time())
 
 ## produce REMIND LCOE reporting *.csv based on gdx information
-if (! isTRUE(envir$cfg$gms$c_empty_model == "on") || ! grepl("^C_TESTTHAT", scenario)) {
-  message("### start generation of LCOE reporting at ", Sys.time())
-  tmp <- try(convGDX2CSV_LCOE(gdx,file=LCOE_reporting_file,scen=scenario)) # execute convGDX2MIF_LCOE
-  message("### end generation of LCOE reporting at ", Sys.time())
-}
+message("### start generation of LCOE reporting at ", Sys.time())
+tmp <- try(convGDX2CSV_LCOE(gdx,file=LCOE_reporting_file,scen=scenario)) # execute convGDX2MIF_LCOE
+message("### end generation of LCOE reporting at ", Sys.time())
 
 ## generate DIETER reporting if it is needed
 ## the reporting is appended to REMIND_generic_<scenario>.MIF in "DIETER" Sub Directory

scripts/output/single/reportingREMIND2MAgPIE.R

@@ -21,36 +21,35 @@ if(!exists("source_include")) {
 
 gdx      <- file.path(outputdir,gdx_name)
 scenario <- getScenNames(outputdir)
+
+configfile <- file.path(outputdir, "config.Rdata")
+envir <- new.env()
+load(configfile, envir = envir)
+
 ###############################################################################
 # paths of the reporting files
 remind_reporting_file <- file.path(outputdir,paste0("REMIND_generic_",scenario,".mif"))
 
 # produce REMIND reporting *.mif based on gdx information
+message("\n### start generation of mif files at ", Sys.time())
 convGDX2MIF_REMIND2MAgPIE(gdx, file = remind_reporting_file, scenario = scenario)
 
-# create common REMIND-MAgPIE reporting by sticking individual REMIND and MAgPIE reporting mifs together
-configfile <- file.path(outputdir, "config.Rdata")
-envir <- new.env()
-load(configfile, envir = envir)
 magpie_reporting_file <- envir$cfg$pathToMagpieReport
 if (! is.null(magpie_reporting_file) && file.exists(magpie_reporting_file)) {
   message("add MAgPIE reporting from ", magpie_reporting_file)
-  tmp_rem <- read.report(remind_reporting_file, as.list=FALSE)
-  tmp_mag <- read.report(magpie_reporting_file, as.list=FALSE)[, getYears(tmp_rem), ]
+  tmp_rem <- quitte::as.quitte(remind_reporting_file)
+  tmp_mag <- dplyr::filter(quitte::as.quitte(magpie_reporting_file), .data$period %in% unique(tmp_rem$period))
   # remove population from magpie reporting to avoid duplication (units "million" vs. "million people")
-  tmp_mag <- tmp_mag[, , "Population (million people)", invert = TRUE]  
-  # harmonize scenario name from -mag-xx to -rem-xx
-  getNames(tmp_mag, dim = 1) <- paste0(scenario)
-  tmp_rem_mag <- mbind(tmp_rem, tmp_mag)
-  # extract variable names without units for both models
-  remind_variables <- magclass::unitsplit(getNames(tmp_rem_mag[, , "REMIND"], dim = 3))$variable
-  magpie_variables <- magclass::unitsplit(getNames(tmp_rem_mag[, , "MAgPIE"], dim = 3))$variable
-  if (any(remind_variables %in% magpie_variables)) {
-      message("Cannot produce common REMIND-MAgPIE reporting because there are identical variable names in both models!")
-  } else {
-    write.report(tmp_rem_mag, file = remind_reporting_file, ndigit = 7)
-    deletePlus(remind_reporting_file, writemif = TRUE)
+  sharedvariables <- intersect(tmp_mag$variable, tmp_rem$variable)
+  if (length(sharedvariables) > 0) {
+    message("The following variables will be dropped from MAgPIE reporting because they are in REMIND reporting: ", paste(sharedvariables, collapse = ", "))
+    tmp_mag <- dplyr::filter(tmp_mag, ! .data$variable %in% sharedvariables)
   }
+  # harmonize scenario name from -mag-xx to -rem-xx
+  tmp_mag$scenario <- paste0(scenario)
+  tmp_rem_mag <- rbind(tmp_rem, tmp_mag)
+  quitte::write.mif(tmp_rem_mag, path = remind_reporting_file)
+  deletePlus(remind_reporting_file, writemif = TRUE)
 }
 
 message("### end generation of mif files at ", Sys.time())

start_bundle_coupled.R

@@ -632,7 +632,7 @@ for (scen in common) {
       }
       slurmOptions <- combine_slurmConfig(paste0("--qos=", runEnv$qos, " --job-name=", fullrunname, " --output=", logfile,
         " --open-mode=append --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks,
-        if (runEnv$numberOfTasks == 1) " --mem=8000"), runEnv$sbatch)
+        if (runEnv$numberOfTasks == 1) " --mem=16000"), runEnv$sbatch)
       slurmCommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"")
       message(slurmCommand)
       exitCode <- system(slurmCommand)

start_coupled.R

@@ -74,7 +74,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
 
     # Switch off generation of needless output for all but the last REMIND iteration
     if (i < max_iterations) {
-      cfg_rem$output <- intersect(c("reporting", "emulator", "rds_report", "reportingREMIND2MAgPIE"), cfg_rem_original)
+      cfg_rem$output <- intersect(cfg_rem_original, c("reporting", "reportingREMIND2MAgPIE", "emulator", "rds_report"))
     } else {
       cfg_rem$output <- cfg_rem_original
     }
@@ -260,7 +260,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
         }
         slurmOptions <- combine_slurmConfig(paste0("--qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile,
            " --open-mode=append --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks,
-          if (subseq.env$numberOfTasks == 1) " --mem=8000"), subseq.env$sbatch)
+          if (subseq.env$numberOfTasks == 1) " --mem=16000"), subseq.env$sbatch)
         subsequentcommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"")
         message(subsequentcommand)
         if (length(needfulldatagdx) > 0) {