Merge pull request #534 from orichters/smallfix

fix and simplify NGFS+SHAPE piamInterfaces test

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '224202816'
+ValidationKey: '224267960'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     -   id: mixed-line-ending
 
 -   repo: https://github.com/lorenzwalthert/precommit
-    rev: v0.3.2.9025
+    rev: v0.4.0
     hooks:
     -   id: parsable-R
     -   id: deps-in-desc

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'remind2: The REMIND R package (2nd generation)'
-version: 1.134.4
-date-released: '2024-02-11'
+version: 1.134.5
+date-released: '2024-02-15'
 abstract: Contains the REMIND-specific routines for data and model output manipulation.
 authors:
 - family-names: Rodrigues

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: remind2
 Title: The REMIND R package (2nd generation)
-Version: 1.134.4
-Date: 2024-02-11
+Version: 1.134.5
+Date: 2024-02-15
 Authors@R: c(
     person("Renato", "Rodrigues", , "[email protected]", role = c("aut", "cre")),
     person("Lavinia", "Baumstark", role = "aut"),

README.md

@@ -1,6 +1,6 @@
 # The REMIND R package (2nd generation)
 
-R package **remind2**, version **1.134.4**
+R package **remind2**, version **1.134.5**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/remind2)](https://cran.r-project.org/package=remind2)  [![R build status](https://github.com/pik-piam/remind2/workflows/check/badge.svg)](https://github.com/pik-piam/remind2/actions) [![codecov](https://codecov.io/gh/pik-piam/remind2/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/remind2) [![r-universe](https://pik-piam.r-universe.dev/badges/remind2)](https://pik-piam.r-universe.dev/builds)
 
@@ -49,7 +49,7 @@ In case of questions / problems please contact Renato Rodrigues <renato.rodrigue
 
 To cite package **remind2** in publications use:
 
-Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Duerrwaechter J, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P, Weigmann P (2024). _remind2: The REMIND R package (2nd generation)_. R package version 1.134.4, <https://github.com/pik-piam/remind2>.
+Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Duerrwaechter J, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P, Weigmann P (2024). _remind2: The REMIND R package (2nd generation)_. R package version 1.134.5, <URL: https://github.com/pik-piam/remind2>.
 
 A BibTeX entry for LaTeX users is
 
@@ -58,7 +58,7 @@ A BibTeX entry for LaTeX users is
   title = {remind2: The REMIND R package (2nd generation)},
   author = {Renato Rodrigues and Lavinia Baumstark and Falk Benke and Jan Philipp Dietrich and Alois Dirnaichner and Jakob Duerrwaechter and Pascal Führlich and Anastasis Giannousakis and Robin Hasse and Jérome Hilaire and David Klein and Johannes Koch and Katarzyna Kowalczyk and Antoine Levesque and Aman Malik and Anne Merfort and Leon Merfort and Simón Morena-Leiva and Michaja Pehl and Robert Pietzcker and Sebastian Rauner and Oliver Richters and Marianna Rottoli and Christof Schötz and Felix Schreyer and Kais Siala and Björn Sörgel and Mike Spahr and Jessica Strefler and Philipp Verpoort and Pascal Weigmann},
   year = {2024},
-  note = {R package version 1.134.4},
+  note = {R package version 1.134.5},
   url = {https://github.com/pik-piam/remind2},
 }
 ```

tests/testthat/test-convGDX2mif.R

@@ -27,37 +27,66 @@ test_that("Test if REMIND reporting is produced as it should and check data inte
     gdxPaths <- defaultGdxPath
   }
 
-  # finds for each AMT scenario the most recent, successfully converged GDX
-  .findAMTgdx <- function(gdxPaths = NULL) {
-    didremindfinish <- function(fulldatapath) {
-      logpath <- paste0(stringr::str_sub(fulldatapath, 1, -14), "/full.log")
-      return(file.exists(logpath) &&
-        any(grep("*** Status: Normal completion", readLines(logpath, warn = FALSE), fixed = TRUE)))
+  # finds for each AMT scenario the most recent, successfully converged GDX,
+  # that is no older than 30 days
+  .findAMTgdx <- function(gdxPaths = NULL, scenario = NULL) {
+    # regex for "%Y-%m-%d_%H.%M.%S" timestamp
+    datetimepattern <- "[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}"
+
+    .did_REMIND_finish <- function(path) {
+      logpath <- sub("fulldata.gdx$", "full.log", path)
+      return(   file.exists(logpath)
+             && "*** Status: Normal completion" %in% readLines(logpath,
+                                                               warn = FALSE))
     }
-    gdx <- Sys.glob("/p/projects/remind/modeltests/output/*/fulldata.gdx")
-    stamp <- lapply(gdx, stringr::str_sub, -32, -14) %>%
-      strptime(format = "%Y-%m-%d_%H.%M.%S") %>%
-      as.numeric()
-    gdx <- data.frame(list(gdx = gdx, stamp = stamp))
-    gdx <- gdx[Sys.time() - gdx$stamp < 30 * 24 * 60 * 60 & !is.na(gdx$stamp), ]
-    gdx <- gdx[unlist(lapply(gdx$gdx, didremindfinish)), ]
-    gdx <- gdx[order(gdx$stamp), ]
-    datetimepattern <- "_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}"
-    gdx$scenario <- sub(datetimepattern, "", basename(dirname(as.vector(gdx$gdx))))
-    for (scenarioname in unique(gdx$scenario)) {
-      gdxPaths <- c(gdxPaths, as.character(tail(gdx[gdx$scenario == scenarioname, ]$gdx, n = 1)))
+
+    .latest_run_of_scenario <- function(path) {
+      # sort by scenario and decreasing time, latest runs come first
+      path <- path %>%
+        sort(decreasing = TRUE)
+
+      # duplicates are older than the preceding runs
+      dup <- duplicated(sub(paste0("_", datetimepattern), "",
+                            basename(dirname(path))))
+
+      # everything that is no duplicate is the latest run of that scenario
+      return(path[!dup])
+    }
+
+    return(c(gdxPaths,
+             Sys.glob(paste0("/p/projects/remind/modeltests/remind/output/",
+                             scenario, "*/fulldata.gdx")) %>%
+               Filter(.did_REMIND_finish, x = .) %>%
+               .latest_run_of_scenario()))
+  }
+
+  checkPiamTemplates <- function(computedVariables) {
+    templates <- c("AR6", "AR6_NGFS", "NAVIGATE", "SHAPE", "ELEVATE")
+    for (template in templates) {
+      templateVariables <- template %>%
+        piamInterfaces::getREMINDTemplateVariables() %>%
+        unique() %>%
+        deletePlus()
+      expect_true(any(computedVariables %in% templateVariables))
+      missingVariables <- setdiff(templateVariables, computedVariables)
+      if (length(missingVariables) > 0) {
+        warning("The following variables are expected in the piamInterfaces package ",
+                "for template ", template,
+                ", but cannot be found in the reporting generated by ", gdxPath, ":\n ",
+                paste(missingVariables, collapse = ",\n "))
+      }
     }
-    return(gdxPaths)
   }
 
   # uncomment to add current calibration gdxes
   # gdxPaths <- c(gdxPaths, Sys.glob("/p/projects/remind/inputdata/CESparametersAndGDX/*.gdx"))
   # uncomment to add debugging example gdx files
   # gdxPaths <- c(gdxPaths, Sys.glob("/p/projects/remind/debugging/gdx-examples/*.gdx"))
   # uncomment to add gdx files from most recent AMT runs
-  # gdxPaths <- c(gdxPaths, .findAMTgdx())
+  gdxPaths <- c(gdxPaths, .findAMTgdx(scenario = "SSP2EU-NPi-AMT"))
 
   numberOfMifs <- 0
+
   for (gdxPath in gdxPaths) {
     numberOfMifs <- numberOfMifs + 1
 
@@ -66,6 +95,9 @@ test_that("Test if REMIND reporting is produced as it should and check data inte
 
     expect_no_warning(checkVariableNames(getNames(mifContent, dim = 3)))
 
+    computedVariables <- deletePlus(getItems(mifContent, dim = 3.3))
+    computedVariables <- gsub("\\(\\)", "(unitless)", computedVariables)
+    checkPiamTemplates(computedVariables)
     magclass::write.report(
       x = magclass::collapseNames(mifContent),
       file = file.path(tempdir(), paste0(numberOfMifs, ".mif")),