fix output script bugs

DESCRIPTION

@@ -51,7 +51,7 @@ Imports:
     nleqslv,
     optparse,
     piamenv (>= 0.4.0),
-    piamInterfaces (>= 0.12.18),
+    piamInterfaces (>= 0.13.8),
     plotly,
     purrr,
     quitte (>= 0.3128.4),

config/default.cfg

@@ -119,6 +119,9 @@ cfg$logoption <- 2
 cfg$output <- c("reporting","reportCEScalib","rds_report","fixOnRef","checkProjectSummations")
 # "ar6Climate","validation","emulator","reportCEScalib","validationSummary","dashboard"
 
+# automatically fix remaining differences between run and path_ref
+cfg$fixOnRefAuto <- FALSE
+
 # Set the format for the results folder, type string :date: in order to use the current time stamp in the folder name (e.g. "results:date:") use :title: to use the current title in the folder name
 cfg$results_folder <- "output/:title::date:"
 

output.R

@@ -93,8 +93,12 @@ choose_slurmConfig_output <- function(output) {
     return("direct")
 
   # Modify slurm options for ar6 reporting, since we want to run MAGICC in parallel and we'll need a lot of memory
-  if ("ar6Climate" %in% output) slurm_options <- paste(slurm_options[1:3], "--tasks-per-node=12 --mem=32000")
-  # reporting.R, in particular remind2::convGDX2MIF, requires at least --mem=8000 of memory
+  if ("ar6Climate" %in% output) {
+    slurm_options <- paste(slurm_options[1:3], "--tasks-per-node=12 --mem=32000")
+  } else if ("nashAnalysis" %in% output) {
+    slurm_options <- paste(slurm_options[1:3], "--mem=32000")
+  }
+  # reporting.R, in particular remind2::convGDX2MIF, requires at least --mem=8000
   if ("reporting" %in% output) slurm_options <- grep("--mem=[0-9]*[0-9]{3}", slurm_options, value = TRUE)
 
   if (length(slurm_options) == 1) {
@@ -275,12 +279,13 @@ if (comp %in% c("comparison", "export")) {
     } else {
       message("\nStarting output generation for ", outputdir, "\n")
       name <- paste0(output, ".R")
+      scriptsfound <- file.exists(paste0("scripts/output/single/", name))
       if ("--test" %in% flags) {
         message("Test mode, not executing scripts/output/single/", paste(name, collapse = ", "))
-      } else if (all(file.exists(paste0("scripts/output/single/", name)))) {
+      } else {
         if (slurmConfig == "direct") {
           # execute output script directly (without sending it to slurm)
-          for (n in name) {
+          for (n in name[scriptsfound]) {
             message("Executing ", n)
             tmp.env <- new.env()
             tmp.error <- try(sys.source(paste0("scripts/output/single/", n), envir = tmp.env))
@@ -302,9 +307,10 @@ if (comp %in% c("comparison", "export")) {
         }
         # finished
         message("\nFinished ", ifelse(slurmConfig == "direct", "", "starting job for "), "output generation for ", outputdir, "!\n")
-      } else {
-        warning("Skipping ", outputdir, " because some output script selected could not be found ",
-                "in scripts/output/single: ", name[! name %in% dir("scripts/output/single")])
+      }
+      if (any(! scriptsfound)) {
+        warning("Skipping those output script selected that could not be found in scripts/output/single: ",
+                name[! scriptsfound])
       }
     }
 

scripts/output/single/ar6Climate.R

@@ -224,6 +224,8 @@ climateAssessmentData <- read.quitte(climateAssessmentOutput) %>%
   interpolate_missing_periods(usePeriods, expand.values = FALSE) %>%
   write.mif(remindReportingFile, append = TRUE)
 
+deletePlus(remind_reporting_file, writemif = TRUE)
+
 logmsg <- paste0(
   date(), " postprocessing done! Results appended to REMIND mif '", remindReportingFile, "'\n",
   "ar6Climate.R finished\n"

scripts/output/single/checkProjectSummations.R

@@ -32,6 +32,7 @@ for (template in c("AR6", "NAVIGATE")) {
                      paste("\n-", unique(failing$variable), collapse = ""))
   }
 }
+
 if (length(stopmessage) > 0) {
   stop("Failing summation checks, see above.", stopmessage)
 }

scripts/output/single/fixOnRef.R

@@ -37,7 +37,7 @@ findRefMif <- function(outputdir, envi) {
   return(refmif)
 }
 
-fixMAGICC <- function(d, dref, startyear) {
+fixMAGICC <- function(d, dref, startyear, scens) {
   magiccgrep <- "^Forcing|^Temperature|^Concentration"
   message("Fixing MAGICC6 data before ", startyear)
   dnew <-
@@ -46,7 +46,9 @@ fixMAGICC <- function(d, dref, startyear) {
              .data$period < startyear),
       filter(d, ! grepl(magiccgrep, .data$variable) |
              .data$period >= startyear)
-    )
+    ) %>%
+    mutate(scenario = factor(scens)) %>%
+    droplevels()
   return(dnew)
 }
 
@@ -77,21 +79,23 @@ fixOnMif <- function(outputdir) {
   refname <- basename(dirname(refmif))
   d <- quitte::as.quitte(mifs)
   dref <- quitte::as.quitte(refmif)
-  d <- fixMAGICC(d, dref, startyear)
+  d <- fixMAGICC(d, dref, startyear, scens)
   failfile <- file.path(outputdir, "log_fixOnRef.csv")
-  fixeddata <- piamInterfaces::fixOnRef(d, dref, ret = "fixed", startyear = startyear, failfile = failfile)
+  fixeddata <- piamInterfaces::fixOnRef(d, dref, ret = "TRUE_or_fixed", startyear = startyear, failfile = failfile)
 
-  if (exists("flags") && isTRUE("--interactive" %in% flags)) {
-    message("\nDo you want to fix that by overwriting ", title, " mif with reference run ", refname, " for t < ", startyear, "?\nType: y/N")
-    if (tolower(gms::getLine()) %in% c("y", "yes")) {
-      message("Updating ", mifs[[1]])
-      tmpfile <- paste0(mifs[[1]], "fixOnMif")
-      quitte::write.mif(fixeddata, tmpfile)
-      file.rename(tmpfile, mifs[[1]])
-      remind2::deletePlus(mifs[[1]], writemif = TRUE)
-      message("Keep in mind to update the runs that use this as `path_gdx_ref` as well.")
-    }
+  if (! isTRUE(fixeddata) && isTRUE(envi$cfg$fixOnRefAuto)) {
+    d <- fixeddata
+  } else if (! isTRUE(fixeddata) && exists("flags") && isTRUE("--interactive" %in% flags)) {
+    message("\nDo you want to fix that by overwriting ", title, " mif with reference run ",
+            refname, " for t < ", startyear, "?\nType: y/N")
+    if (tolower(gms::getLine()) %in% c("y", "yes")) d <- fixeddata
   }
+  message("Updating ", mifs[[1]])
+  tmpfile <- paste0(mifs[[1]], "fixOnMif")
+  quitte::write.mif(d, tmpfile)
+  file.rename(tmpfile, mifs[[1]])
+  remind2::deletePlus(mifs[[1]], writemif = TRUE)
+  message("Keep in mind to update the runs that use this as `path_gdx_ref` as well.")
   return(NULL)
 }