From 14fce4a6a70f008a25cc8f02b5c1114270fd4405 Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Thu, 5 Sep 2024 10:09:14 +0200
Subject: [PATCH 1/8] filter IEA ETP production data the madrat way

---
 R/calcIEA_ETP.R                              | 17 ++++-
 R/filter_historical_mif.R                    | 71 --------------------
 R/fullVALIDATIONREMIND.R                     |  3 -
 inst/extdata/historical_mif_filter_table.csv |  7 --
 4 files changed, 15 insertions(+), 83 deletions(-)
 delete mode 100644 R/filter_historical_mif.R
 delete mode 100644 inst/extdata/historical_mif_filter_table.csv

diff --git a/R/calcIEA_ETP.R b/R/calcIEA_ETP.R
index a3ef5a5b..88619468 100644
--- a/R/calcIEA_ETP.R
+++ b/R/calcIEA_ETP.R
@@ -11,7 +11,6 @@
 #' @export
 
 calcIEA_ETP <- function() {
-
   mapping <- toolGetMapping("Mapping_IEA_ETP.csv", type = "reportingVariables", where = "mrremind") %>%
     filter(!is.na(!!sym("REMIND")), !!sym("REMIND") != "") %>%
     mutate("Conversion" = as.numeric(!!sym("Conversion"))) %>%
@@ -52,11 +51,25 @@ calcIEA_ETP <- function() {
   x <- aggregate(value ~ region + year + model + variable, x, sum, na.action = na.pass) %>%
     as.magpie()
 
+  .customAggregate <- function(x, rel, to = NULL) {
+    out <- toolAggregate(x, rel = rel, to = to)
+
+    # remove cement and industry production for all regions except for GLO
+    if ("GLO" %in% getItems(out, dim = 1)) {
+      out[setdiff(getItems(out, dim = 1), "GLO"), ,
+          c("Production|Industry|Cement (Mt/yr)", "Production|Industry|Steel (Mt/yr)")] <- NA
+
+    } else {
+      out <- out[, , c("Production|Industry|Cement (Mt/yr)", "Production|Industry|Steel (Mt/yr)"), invert = TRUE]
+    }
+    return(out)
+  }
+
   return(list(
     x = x,
     weight = NULL,
+    aggregationFunction = .customAggregate,
     unit = c("EJ/yr", "Mt CO2/yr", "Mt/yr", "bn pkm/yr", "bn tkm/yr"),
     description = "IEA ETP projections as REMIND variables"
   ))
-
 }
diff --git a/R/filter_historical_mif.R b/R/filter_historical_mif.R
deleted file mode 100644
index c86d3f27..00000000
--- a/R/filter_historical_mif.R
+++ /dev/null
@@ -1,71 +0,0 @@
-#' Filter improperly aggregated regional data from historical.mif
-#'
-#' Load a `historical.mif` file, remove designated data, write back to file.
-#'
-#' For all combinations of scenario, model, and variable (and all years), either
-#' all regions listed in `exclude_regions` in the `filter_table` are removed, or
-#' all regions _except_ those listed in `include_regions` in the `filter_table`
-#' are removed.
-#' Defining both `include_regions` and `exclude_regions` on the same row of
-#' `filter_table` will throw an error.
-#' Scenarios, models, and variables are matched precisely, not partially or via
-#' regular expressions.
-#'
-#' @md
-#' @param path Path to `historical.mif` (or any mif for that matter), defaults
-#'   to `historical.mif` in the currently configured
-#'   `getConfig('outputfolder')`.
-#' @param filter_table Data frame with columns `scenario`, `model`, `variable`,
-#'   `include_regions`, and `exclude_regions`.  Defaults to the built-in
-#'   `./inst/extdata/historical_mif_filter_table.csv` of the `mrremind` package.
-#'
-#' @importFrom quitte cartesian
-
-#' @export
-filter_historical_mif <- function(path = NULL, filter_table = NULL) {
-    if (is.null(path))
-        path <- 'historical.mif'
-
-    if (is.null(filter_table))
-        filter_table <- read.csv(
-            file = system.file('extdata', 'historical_mif_filter_table.csv',
-                               package = 'mrremind', mustWork = TRUE),
-            colClasses = 'character')
-
-    # combine scenario, model, and variable columns to string for indexing a
-    # magpie object
-    scenario_model_variable <- sapply(
-        seq_len(nrow(filter_table)),
-        function(i) {
-            Reduce(f = cartesian,
-                   x = filter_table[i, c('scenario', 'model', 'variable')])
-        })
-
-    # check for conflicting include/exclude definitions
-    include_and_exclude <- ( '' != filter_table[['include_regions']]
-                           & '' != filter_table[['exclude_regions']])
-
-    if (any(include_and_exclude))
-        stop('Both include and exclude regions defined for:',
-             paste(c('', scenario_model_variable[include_and_exclude]),
-                   collapse = '\n'))
-
-    h <- read.report(file = path, as.list = FALSE)
-
-    # process each row of the filter table
-    for (i in seq_len(nrow(filter_table))) {
-        if ('' != filter_table[[i, 'include_regions']]) {
-          excludedRegions <- setdiff(getItems(h, dim = 'region'),
-                                     filter_table[[i, 'include_regions']])
-        } else {
-          excludedRegions <- filter_table[[i, 'exclude_regions']]
-        }
-
-        h_include <- h[, , scenario_model_variable[i]]
-        h_include[excludedRegions, , ] <- NA
-
-        h <- mbind(h[, , scenario_model_variable[i], invert = TRUE], h_include)
-    }
-
-    write.report(h, file = path)
-}
diff --git a/R/fullVALIDATIONREMIND.R b/R/fullVALIDATIONREMIND.R
index c0974067..249fedc6 100644
--- a/R/fullVALIDATIONREMIND.R
+++ b/R/fullVALIDATIONREMIND.R
@@ -259,7 +259,4 @@ fullVALIDATIONREMIND <- function(rev = 0) {
     try = FALSE, years = years,
     writeArgs = list(scenario = "historical", model = "INDSTAT2")
   )
-
-  # filter variables that are too imprecise on regional level ----
-  filter_historical_mif()
 }
diff --git a/inst/extdata/historical_mif_filter_table.csv b/inst/extdata/historical_mif_filter_table.csv
deleted file mode 100644
index 557c8b1f..00000000
--- a/inst/extdata/historical_mif_filter_table.csv
+++ /dev/null
@@ -1,7 +0,0 @@
-scenario,model,variable,include_regions,exclude_regions
-historical,IEA ETP 2DS,Production|Industry|Cement (Mt/yr),GLO,
-historical,IEA ETP 2DS,Production|Industry|Steel (Mt/yr),GLO,
-historical,IEA ETP B2DS,Production|Industry|Cement (Mt/yr),GLO,
-historical,IEA ETP B2DS,Production|Industry|Steel (Mt/yr),GLO,
-historical,IEA ETP RTS,Production|Industry|Cement (Mt/yr),GLO,
-historical,IEA ETP RTS,Production|Industry|Steel (Mt/yr),GLO,

From 00f315c403062cbda72f724be260d1d16121feb8 Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Fri, 6 Sep 2024 09:39:12 +0200
Subject: [PATCH 2/8] increment version

---
 .buildlibrary                |  2 +-
 CITATION.cff                 |  4 ++--
 DESCRIPTION                  |  4 ++--
 NAMESPACE                    |  1 -
 README.md                    |  6 +++---
 man/filter_historical_mif.Rd | 30 ------------------------------
 6 files changed, 8 insertions(+), 39 deletions(-)
 delete mode 100644 man/filter_historical_mif.Rd

diff --git a/.buildlibrary b/.buildlibrary
index fb6a5ed0..fb846a95 100644
--- a/.buildlibrary
+++ b/.buildlibrary
@@ -1,4 +1,4 @@
-ValidationKey: '37405683'
+ValidationKey: '37427528'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'
diff --git a/CITATION.cff b/CITATION.cff
index 0354a654..9c9395c0 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'mrremind: MadRat REMIND Input Data Package'
-version: 0.187.3
-date-released: '2024-09-05'
+version: 0.187.4
+date-released: '2024-09-06'
 abstract: The mrremind packages contains data preprocessing for the REMIND model.
 authors:
 - family-names: Baumstark
diff --git a/DESCRIPTION b/DESCRIPTION
index fbd93e5c..f837a249 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,8 +1,8 @@
 Type: Package
 Package: mrremind
 Title: MadRat REMIND Input Data Package
-Version: 0.187.3
-Date: 2024-09-05
+Version: 0.187.4
+Date: 2024-09-06
 Authors@R: c(
     person("Lavinia", "Baumstark", , "lavinia@pik-potsdam.de", role = c("aut", "cre")),
     person("Renato", "Rodrigues", role = "aut"),
diff --git a/NAMESPACE b/NAMESPACE
index 7411a66b..6b44e248 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -48,7 +48,6 @@ export(convertUNFCCC)
 export(convertUNIDO)
 export(convertUSGS)
 export(exportThresholds)
-export(filter_historical_mif)
 export(readADVANCE_WP2)
 export(readAGEB)
 export(readAR6GWP)
diff --git a/README.md b/README.md
index 7ae04cdc..e73f1c97 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # MadRat REMIND Input Data Package
 
-R package **mrremind**, version **0.187.3**
+R package **mrremind**, version **0.187.4**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/mrremind)](https://cran.r-project.org/package=mrremind)  [![R build status](https://github.com/pik-piam/mrremind/workflows/check/badge.svg)](https://github.com/pik-piam/mrremind/actions) [![codecov](https://codecov.io/gh/pik-piam/mrremind/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrremind) [![r-universe](https://pik-piam.r-universe.dev/badges/mrremind)](https://pik-piam.r-universe.dev/builds)
 
@@ -39,7 +39,7 @@ In case of questions / problems please contact Lavinia Baumstark <lavinia@pik-po
 
 To cite package **mrremind** in publications use:
 
-Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.187.3, <https://github.com/pik-piam/mrremind>.
+Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.187.4, <https://github.com/pik-piam/mrremind>.
 
 A BibTeX entry for LaTeX users is
 
@@ -48,7 +48,7 @@ A BibTeX entry for LaTeX users is
   title = {mrremind: MadRat REMIND Input Data Package},
   author = {Lavinia Baumstark and Renato Rodrigues and Antoine Levesque and Julian Oeser and Christoph Bertram and Ioanna Mouratiadou and Aman Malik and Felix Schreyer and Bjoern Soergel and Marianna Rottoli and Abhijeet Mishra and Alois Dirnaichner and Michaja Pehl and Anastasis Giannousakis and David Klein and Jessica Strefler and Lukas Feldhaus and Regina Brecha and Sebastian Rauner and Jan Philipp Dietrich and Stephen Bi and Falk Benke and Pascal Weigmann and Oliver Richters and Robin Hasse and Sophie Fuchs and Rahel Mandaroux and Johannes Koch},
   year = {2024},
-  note = {R package version 0.187.3},
+  note = {R package version 0.187.4},
   url = {https://github.com/pik-piam/mrremind},
 }
 ```
diff --git a/man/filter_historical_mif.Rd b/man/filter_historical_mif.Rd
deleted file mode 100644
index c2063716..00000000
--- a/man/filter_historical_mif.Rd
+++ /dev/null
@@ -1,30 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/filter_historical_mif.R
-\name{filter_historical_mif}
-\alias{filter_historical_mif}
-\title{Filter improperly aggregated regional data from historical.mif}
-\usage{
-filter_historical_mif(path = NULL, filter_table = NULL)
-}
-\arguments{
-\item{path}{Path to \code{historical.mif} (or any mif for that matter), defaults
-to \code{historical.mif} in the currently configured
-\code{getConfig('outputfolder')}.}
-
-\item{filter_table}{Data frame with columns \code{scenario}, \code{model}, \code{variable},
-\code{include_regions}, and \code{exclude_regions}.  Defaults to the built-in
-\code{./inst/extdata/historical_mif_filter_table.csv} of the \code{mrremind} package.}
-}
-\description{
-Load a \code{historical.mif} file, remove designated data, write back to file.
-}
-\details{
-For all combinations of scenario, model, and variable (and all years), either
-all regions listed in \code{exclude_regions} in the \code{filter_table} are removed, or
-all regions \emph{except} those listed in \code{include_regions} in the \code{filter_table}
-are removed.
-Defining both \code{include_regions} and \code{exclude_regions} on the same row of
-\code{filter_table} will throw an error.
-Scenarios, models, and variables are matched precisely, not partially or via
-regular expressions.
-}

From d69deec278d8371e9d12ecf2c358f71f7e7805cc Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Wed, 25 Sep 2024 18:43:10 +0200
Subject: [PATCH 3/8] refactor convertIEA_ETP, dropping unusable regional
 disaggregation

---
 R/calcIEA_ETP.R                               |  75 ++++--
 R/convertIEA_ETP.R                            |  62 +----
 R/readAGEB.R                                  |   2 +-
 R/readIEA_ETP.R                               |   8 +-
 .../extdata/regional/regionmappingIEA_ETP.csv | 250 ++++++++++++++++++
 5 files changed, 321 insertions(+), 76 deletions(-)
 create mode 100755 inst/extdata/regional/regionmappingIEA_ETP.csv

diff --git a/R/calcIEA_ETP.R b/R/calcIEA_ETP.R
index 88619468..c3f4ec4d 100644
--- a/R/calcIEA_ETP.R
+++ b/R/calcIEA_ETP.R
@@ -12,8 +12,8 @@
 
 calcIEA_ETP <- function() {
   mapping <- toolGetMapping("Mapping_IEA_ETP.csv", type = "reportingVariables", where = "mrremind") %>%
-    filter(!is.na(!!sym("REMIND")), !!sym("REMIND") != "") %>%
-    mutate("Conversion" = as.numeric(!!sym("Conversion"))) %>%
+    filter(!is.na(.data$REMIND), .data$REMIND != "") %>%
+    mutate("Conversion" = as.numeric(.data$Conversion)) %>%
     select("variable" = "IEA_ETP", "REMIND", "Conversion", "Unit_REMIND")
 
   mapping$variable <- trimws(mapping$variable)
@@ -24,51 +24,86 @@ calcIEA_ETP <- function() {
   x3 <- readSource("IEA_ETP", subtype = "buildings")
   x4 <- readSource("IEA_ETP", subtype = "summary")
 
-  data <- mbind(x1, x2, x3, x4)
+  x5 <- mbind(
+    readSource("IEA_ETP", subtype = "industry", convert = F)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "transport", convert = F)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "buildings", convert = F)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "summary", convert = F)["WORLD", , ]
+  )
 
-  data <- as.data.frame(data) %>%
+
+  data <- as.data.frame(mbind(x1, x2, x3, x4)) %>%
+    as_tibble() %>%
+    select(
+      "region" = "Region", "scenario" = "Data1", "variable" = "Data2",
+      "year" = "Year", "value" = "Value"
+    ) %>%
+    filter(!is.na(.data$value))
+
+  dataGlo <- as.data.frame(x5) %>%
     as_tibble() %>%
     select(
       "region" = "Region", "scenario" = "Data1", "variable" = "Data2",
       "year" = "Year", "value" = "Value"
-    )
+    ) %>%
+    filter(!is.na(.data$value)) %>%
+    mutate(region = "GLO")
+
+
 
   x <- left_join(
-    data,
+    rbind(data, dataGlo),
     mapping,
     by = "variable",
     relationship = "many-to-many"
   ) %>%
-    filter(!!sym("REMIND") != "") %>%
+    filter(.data$REMIND != "") %>%
     mutate(
-      "value" = !!sym("value") * !!sym("Conversion"),
-      "REMIND" = paste0(!!sym("REMIND"), " (", !!sym("Unit_REMIND"), ")"),
-      "model" = paste0("IEA ETP ", !!sym("scenario")),
-      "year" = as.numeric(as.character(!!sym("year")))
+      "value" = .data$value * .data$Conversion,
+      "REMIND" = paste0(.data$REMIND, " (", .data$Unit_REMIND, ")"),
+      "model" = paste0("IEA ETP ", .data$scenario),
+      "year" = as.numeric(as.character(.data$year))
     ) %>%
     select("region", "year", "model", "variable" = "REMIND", "value")
 
-  x <- aggregate(value ~ region + year + model + variable, x, sum, na.action = na.pass) %>%
+  x <- aggregate(value ~ region + year + model + variable, x, sum) %>%
     as.magpie()
 
-  .customAggregate <- function(x, rel, to = NULL) {
-    out <- toolAggregate(x, rel = rel, to = to)
+  dataGlo <- x["GLO", , ]
+  x <- x["GLO", , invert = T] %>%
+    toolCountryFill(fill = NA, verbosity = 2)
+
+
+  # includes global values from the original source instead of calculating
+  # them as the sum of all countries (as countries are incomplete)
+  .customAggregate <- function(x, rel, to = NULL, glo) {
+    x <- toolAggregate(x, rel = rel, to = to)
+
+    if ("GLO" %in% getItems(x, dim = 1)) {
+      out <- new.magpie(
+        cells_and_regions = getItems(x, dim = 1),
+        years = union(getYears(x), getYears(glo)),
+        names = union(getNames(x), getNames(glo)),
+        fill = NA,
+        sets = names(dimnames(x))
+      )
+
+      x <- x["GLO", , , invert = TRUE]
 
-    # remove cement and industry production for all regions except for GLO
-    if ("GLO" %in% getItems(out, dim = 1)) {
-      out[setdiff(getItems(out, dim = 1), "GLO"), ,
-          c("Production|Industry|Cement (Mt/yr)", "Production|Industry|Steel (Mt/yr)")] <- NA
+      out[getItems(x, dim = 1), getYears(x), getNames(x)] <- x
+      out["GLO", getYears(glo), getNames(glo)] <- glo
 
+      return(out)
     } else {
-      out <- out[, , c("Production|Industry|Cement (Mt/yr)", "Production|Industry|Steel (Mt/yr)"), invert = TRUE]
+      return(x)
     }
-    return(out)
   }
 
   return(list(
     x = x,
     weight = NULL,
     aggregationFunction = .customAggregate,
+    aggregationArguments = list(glo = dataGlo),
     unit = c("EJ/yr", "Mt CO2/yr", "Mt/yr", "bn pkm/yr", "bn tkm/yr"),
     description = "IEA ETP projections as REMIND variables"
   ))
diff --git a/R/convertIEA_ETP.R b/R/convertIEA_ETP.R
index 60d5738f..08ee1da9 100644
--- a/R/convertIEA_ETP.R
+++ b/R/convertIEA_ETP.R
@@ -6,68 +6,28 @@
 #'
 convertIEA_ETP <- function(x, subtype) {
 
-  getItems(x, dim = 1) <- lapply(getItems(x, dim = 1), function(y) ifelse(y == "NonOECD", "Non-OECD", y))
-
-  regmapping <- toolGetMapping("regionmappingIEA_ETP.csv", where = "mappingfolder", type = "regional")
-
+  map <- toolGetMapping("regionmappingIEA_ETP.csv", where = "mrremind", type = "regional")
   fe <- calcOutput("FE", source = "IEA", aggregate = FALSE)
 
-  v <- magpply(x[c("OECD", "Non-OECD"), , , invert = T], function(y) all(is.na(y)), MARGIN = 3)
-  v.oecd.only <- getNames(x[, , v])
-
-  if (is.null(v.oecd.only)) {
-    v.full <- getNames(x)
-  } else {
-    v.full <- getNames(x[, , v.oecd.only, invert = T])
-  }
-
   # disaggregate ASEAN
-  x.asean <- x["ASEAN", , v.full]
-  m <- select(regmapping, c("EEAReg", "CountryCode")) %>% filter(!!sym("EEAReg") == "ASEAN")
+  x.asean <- x["ASEAN", , ]
+  m <- select(map, c("EEAReg", "CountryCode")) %>% filter(.data$EEAReg == "ASEAN")
   w <- fe[m$CountryCode, 2005, "FE|Transport (EJ/yr)"]
   x.asean <- toolAggregate(x.asean, m, from = "EEAReg", to = "CountryCode", weight = w)
 
   # disaggregate European Union
-  x.eu <- x["European Union", , v.full]
-  m <- select(regmapping, c("EEAReg", "CountryCode")) %>% filter(!!sym("EEAReg") == "EUR")
+  x.eu <- x["European Union", , ]
+  m <- select(map, c("EEAReg", "CountryCode")) %>% filter(.data$EEAReg == "EUR")
   w <- fe[m$CountryCode, 2005, "FE|Transport (EJ/yr)"]
   x.eu <- toolAggregate(x.eu, m, from = "EEAReg", to = "CountryCode", weight = w)
 
-  # entries that don't require disaggregation
-  x.ctry <- x[c("Brazil", "China", "India", "Mexico", "Russia", "South Africa", "United States"), , v.full]
-  getItems(x.ctry, dim = 1) <- toolCountry2isocode(getItems(x.ctry, dim = 1), warn = F)
-
-  # disaggregate OECD data for variables with both OECD and finer regional granularity
-  x.oecd.other <- x[c("OECD", "Non-OECD"), , v.full]
-
-  # get OECD/Non-OECD values not accounted for in other regions
-  x.oecd.other["OECD", , ] <- x.oecd.other["OECD", , ] -
-    dimSums(x[c("European Union", "Mexico", "United States"), , v.full], dim = 1)
-  x.oecd.other["Non-OECD", , ] <- x.oecd.other["Non-OECD", , ] -
-    dimSums(x[c("ASEAN", "Brazil", "China", "India", "Russia", "South Africa"), , v.full], dim = 1)
-  m <- select(regmapping, c("OECD", "CountryCode")) %>%
-    filter(!(!!sym("CountryCode") %in% c(getItems(x.asean, dim = 1), getItems(x.eu, dim = 1), getItems(x.ctry, dim = 1))))
-  w <- fe[m$CountryCode, 2005, "FE|Transport (EJ/yr)"]
-  x.oecd.other <- toolAggregate(x.oecd.other, m, from = "OECD", to = "CountryCode", weight = w)
-
-  x.full <- new.magpie(getISOlist(), getYears(x), names = getNames(x), fill = 0)
+  # transform entries that don't require disaggregation
+  x.ctry <- x[c("Brazil", "China", "India", "Mexico", "Russia", "South Africa", "United States"), , ]
+  getItems(x.ctry, dim = 1) <- toolCountry2isocode(getItems(x.ctry, dim = 1), warn = FALSE)
 
-  # for variables with only OECD/nonOECD data we disaggregate this to country-level
-  if (!is.null(v.oecd.only)) {
-    w <- fe[regmapping$CountryCode, 2005, "FE|Transport (EJ/yr)"]
-    x.oecd <- x[c("OECD", "Non-OECD"), , v.oecd.only]
-    x.oecd <- toolAggregate(x.oecd, regmapping, from = "OECD", to = "CountryCode", weight = w)
-    x.full[getItems(x.oecd, dim = 1), , v.oecd.only] <- x.oecd
-  }
 
-  # for variables with OECD/nonOECD data and addtl. region and country data
-  # 1) we disaggregate the more fine-granular regions ASEAN and European Union first
-  # 2) then calculate the values for the OECD/nonOECD regions minus the values for explicitly listed countries/regions
-  # and disaggregate them to the countries not listed explicitly
-  x.full[getItems(x.asean, dim = 1), , v.full] <- x.asean
-  x.full[getItems(x.eu, dim = 1), , v.full] <- x.eu
-  x.full[getItems(x.ctry, dim = 1), , v.full] <- x.ctry
-  x.full[getItems(x.oecd.other, dim = 1), , v.full] <- x.oecd.other
+  x <- mbind(x.asean, x.eu, x.ctry)
+  x <- toolCountryFill(x, fill = NA, verbosity = 2)
 
-  return(x.full)
+  return(x)
 }
diff --git a/R/readAGEB.R b/R/readAGEB.R
index 8e63d354..57f78e9c 100644
--- a/R/readAGEB.R
+++ b/R/readAGEB.R
@@ -84,7 +84,7 @@ readAGEB <- function(subtype = "balances") {
   } else if (subtype == "electricity") {
     read_xlsx(
       path = "STRERZ_Abg_02_2024_korr.xlsx",
-      sheet = "STRERZ (brutto)", col_names = TRUE,
+      sheet = "STRERZ (netto)", col_names = TRUE,
       col_types = c("text", rep("numeric", 34)),
       range = "B3:AJ17", .name_repair = "minimal", na = c("k.A.")
     ) %>%
diff --git a/R/readIEA_ETP.R b/R/readIEA_ETP.R
index ee3adbc2..1de20602 100644
--- a/R/readIEA_ETP.R
+++ b/R/readIEA_ETP.R
@@ -19,7 +19,7 @@ readIEA_ETP <- function(subtype) {
         file = "ETP2017_industry_summary.xlsx",
         prefix = "Industry",
         sheets = list(
-          "OECD", "Non-OECD",
+          "WORLD", "OECD", "Non-OECD",
           "ASEAN", "Brazil", "China", "European Union",
           "India", "Mexico", "Russia", "South Africa", "United States"
         ),
@@ -149,7 +149,7 @@ readIEA_ETP <- function(subtype) {
         file = "ETP2017_buildings_summary.xlsx",
         prefix = "Buildings",
         sheets = list(
-          "OECD", "NonOECD",
+          "WORLD", "OECD", "NonOECD",
           "ASEAN", "Brazil", "China", "European Union",
           "India", "Mexico", "Russia", "South Africa", "United States"
         ),
@@ -278,7 +278,7 @@ readIEA_ETP <- function(subtype) {
         file = "ETP2017_scenario_summary.xlsx",
         prefix = "Summary",
         sheets = list(
-          "OECD", "NonOECD",
+          "WORLD", "OECD", "NonOECD",
           "ASEAN", "Brazil", "China", "European Union",
           "India", "Mexico", "Russia", "South Africa", "United States"
         ),
@@ -452,7 +452,7 @@ readIEA_ETP <- function(subtype) {
         file = "ETP2017_transport_summary.xlsx",
         prefix = "Transport",
         sheets = list(
-          "OECD", "Non-OECD",
+          "WORLD", "OECD", "Non-OECD",
           "ASEAN", "Brazil", "China", "European Union",
           "India", "Mexico", "Russia", "South Africa", "United States"
         ),
diff --git a/inst/extdata/regional/regionmappingIEA_ETP.csv b/inst/extdata/regional/regionmappingIEA_ETP.csv
new file mode 100755
index 00000000..7331c048
--- /dev/null
+++ b/inst/extdata/regional/regionmappingIEA_ETP.csv
@@ -0,0 +1,250 @@
+X;CountryCode;EEAReg;OECD
+Afghanistan;AFG;rest;Non-OECD
+Aland Islands;ALA;EUR;Non-OECD
+Albania;ALB;rest;Non-OECD
+Algeria;DZA;rest;Non-OECD
+American Samoa;ASM;rest;Non-OECD
+Andorra;AND;rest;Non-OECD
+Angola;AGO;rest;Non-OECD
+Anguilla;AIA;rest;Non-OECD
+Antarctica;ATA;rest;Non-OECD
+Antigua and Barbuda;ATG;rest;Non-OECD
+Argentina;ARG;rest;Non-OECD
+Armenia;ARM;rest;Non-OECD
+Aruba;ABW;rest;Non-OECD
+Australia;AUS;rest;OECD
+Austria;AUT;EUR;OECD
+Azerbaijan;AZE;rest;Non-OECD
+Bahamas;BHS;rest;Non-OECD
+Bahrain;BHR;rest;Non-OECD
+Bangladesh;BGD;rest;Non-OECD
+Barbados;BRB;rest;Non-OECD
+Belarus;BLR;rest;Non-OECD
+Belgium;BEL;EUR;OECD
+Belize;BLZ;rest;Non-OECD
+Benin;BEN;rest;Non-OECD
+Bermuda;BMU;rest;Non-OECD
+Bhutan;BTN;rest;Non-OECD
+Bolivia, Plurinational State of;BOL;rest;Non-OECD
+Bonaire, Sint Eustatius and Saba;BES;rest;Non-OECD
+Bosnia and Herzegovina;BIH;rest;Non-OECD
+Botswana;BWA;rest;Non-OECD
+Bouvet Island;BVT;rest;Non-OECD
+Brazil;BRA;BRA;Non-OECD
+British Indian Ocean Territory;IOT;rest;Non-OECD
+Brunei Darussalam;BRN;ASEAN;Non-OECD
+Bulgaria;BGR;EUR;Non-OECD
+Burkina Faso;BFA;rest;Non-OECD
+Burundi;BDI;rest;Non-OECD
+Cambodia;KHM;ASEAN;Non-OECD
+Cameroon;CMR;rest;Non-OECD
+Canada;CAN;rest;OECD
+Cape Verde;CPV;rest;Non-OECD
+Cayman Islands;CYM;rest;Non-OECD
+Central African Republic;CAF;rest;Non-OECD
+Chad;TCD;rest;Non-OECD
+Chile;CHL;rest;OECD
+China;CHN;CHN;Non-OECD
+Christmas Island;CXR;rest;Non-OECD
+Cocos (Keeling) Islands;CCK;rest;Non-OECD
+Colombia;COL;rest;OECD
+Comoros;COM;rest;Non-OECD
+Congo;COG;rest;Non-OECD
+Congo, the Democratic Republic of the;COD;rest;Non-OECD
+Cook Islands;COK;rest;Non-OECD
+Costa Rica;CRI;rest;Non-OECD
+Cote d Ivoire;CIV;rest;Non-OECD
+Croatia;HRV;EUR;Non-OECD
+Cuba;CUB;rest;Non-OECD
+Curacao;CUW;rest;Non-OECD
+Cyprus;CYP;EUR;Non-OECD
+Czech Republic;CZE;EUR;OECD
+Denmark;DNK;EUR;OECD
+Djibouti;DJI;rest;Non-OECD
+Dominica;DMA;rest;Non-OECD
+Dominican Republic;DOM;rest;Non-OECD
+Ecuador;ECU;rest;Non-OECD
+Egypt;EGY;rest;Non-OECD
+El Salvador;SLV;rest;Non-OECD
+Equatorial Guinea;GNQ;rest;Non-OECD
+Eritrea;ERI;rest;Non-OECD
+Estonia;EST;EUR;OECD
+Ethiopia;ETH;rest;Non-OECD
+Falkland Islands (Malvinas);FLK;rest;Non-OECD
+Faroe Islands;FRO;EUR;Non-OECD
+Fiji;FJI;rest;Non-OECD
+Finland;FIN;EUR;OECD
+France;FRA;EUR;OECD
+French Guiana;GUF;rest;Non-OECD
+French Polynesia;PYF;rest;Non-OECD
+French Southern Territories;ATF;rest;Non-OECD
+Gabon;GAB;rest;Non-OECD
+Gambia;GMB;rest;Non-OECD
+Georgia;GEO;rest;Non-OECD
+Germany;DEU;EUR;OECD
+Ghana;GHA;rest;Non-OECD
+Gibraltar;GIB;EUR;Non-OECD
+Greece;GRC;EUR;OECD
+Greenland;GRL;rest;Non-OECD
+Grenada;GRD;rest;Non-OECD
+Guadeloupe;GLP;rest;Non-OECD
+Guam;GUM;rest;Non-OECD
+Guatemala;GTM;rest;Non-OECD
+Guernsey;GGY;EUR;Non-OECD
+Guinea;GIN;rest;Non-OECD
+Guinea-Bissau;GNB;rest;Non-OECD
+Guyana;GUY;rest;Non-OECD
+Haiti;HTI;rest;Non-OECD
+Heard Island and McDonald Islands;HMD;rest;Non-OECD
+Holy See (Vatican City State);VAT;rest;Non-OECD
+Honduras;HND;rest;Non-OECD
+Hong Kong;HKG;rest;Non-OECD
+Hungary;HUN;EUR;OECD
+Iceland;ISL;rest;OECD
+India;IND;IND;Non-OECD
+Indonesia;IDN;ASEAN;Non-OECD
+Iran, Islamic Republic of;IRN;rest;Non-OECD
+Iraq;IRQ;rest;Non-OECD
+Ireland;IRL;EUR;OECD
+Isle of Man;IMN;EUR;Non-OECD
+Israel;ISR;rest;OECD
+Italy;ITA;EUR;OECD
+Jamaica;JAM;rest;Non-OECD
+Japan;JPN;rest;OECD
+Jersey;JEY;EUR;Non-OECD
+Jordan;JOR;rest;Non-OECD
+Kazakhstan;KAZ;rest;Non-OECD
+Kenya;KEN;rest;Non-OECD
+Kiribati;KIR;rest;Non-OECD
+Korea, Democratic People's Republic of;PRK;rest;Non-OECD
+Korea, Republic of;KOR;rest;OECD
+Kuwait;KWT;rest;Non-OECD
+Kyrgyzstan;KGZ;rest;Non-OECD
+Lao People's Democratic Republic;LAO;ASEAN;Non-OECD
+Latvia;LVA;EUR;OECD
+Lebanon;LBN;rest;Non-OECD
+Lesotho;LSO;rest;Non-OECD
+Liberia;LBR;rest;Non-OECD
+Libya;LBY;rest;Non-OECD
+Liechtenstein;LIE;rest;Non-OECD
+Lithuania;LTU;EUR;OECD
+Luxembourg;LUX;EUR;OECD
+Macao;MAC;rest;Non-OECD
+Macedonia, the former Yugoslav Republic of;MKD;rest;Non-OECD
+Madagascar;MDG;rest;Non-OECD
+Malawi;MWI;rest;Non-OECD
+Malaysia;MYS;ASEAN;Non-OECD
+Maldives;MDV;rest;Non-OECD
+Mali;MLI;rest;Non-OECD
+Malta;MLT;EUR;Non-OECD
+Marshall Islands;MHL;rest;Non-OECD
+Martinique;MTQ;rest;Non-OECD
+Mauritania;MRT;rest;Non-OECD
+Mauritius;MUS;rest;Non-OECD
+Mayotte;MYT;rest;Non-OECD
+Mexico;MEX;MEX;OECD
+Micronesia, Federated States of;FSM;rest;Non-OECD
+Moldova, Republic of;MDA;rest;Non-OECD
+Monaco;MCO;rest;Non-OECD
+Mongolia;MNG;rest;Non-OECD
+Montenegro;MNE;rest;Non-OECD
+Montserrat;MSR;rest;Non-OECD
+Morocco;MAR;rest;Non-OECD
+Mozambique;MOZ;rest;Non-OECD
+Myanmar;MMR;ASEAN;Non-OECD
+Namibia;NAM;rest;Non-OECD
+Nauru;NRU;rest;Non-OECD
+Nepal;NPL;rest;Non-OECD
+Netherlands;NLD;EUR;OECD
+New Caledonia;NCL;rest;Non-OECD
+New Zealand;NZL;rest;OECD
+Nicaragua;NIC;rest;Non-OECD
+Niger;NER;rest;Non-OECD
+Nigeria;NGA;rest;Non-OECD
+Niue;NIU;rest;Non-OECD
+Norfolk Island;NFK;rest;Non-OECD
+Northern Mariana Islands;MNP;rest;Non-OECD
+Norway;NOR;rest;OECD
+Oman;OMN;rest;Non-OECD
+Pakistan;PAK;rest;Non-OECD
+Palau;PLW;rest;Non-OECD
+Palestine, State of;PSE;rest;Non-OECD
+Panama;PAN;rest;Non-OECD
+Papua New Guinea;PNG;rest;Non-OECD
+Paraguay;PRY;rest;Non-OECD
+Peru;PER;rest;Non-OECD
+Philippines;PHL;ASEAN;Non-OECD
+Pitcairn;PCN;rest;Non-OECD
+Poland;POL;EUR;OECD
+Portugal;PRT;EUR;OECD
+Puerto Rico;PRI;rest;Non-OECD
+Qatar;QAT;rest;Non-OECD
+Reunion;REU;rest;Non-OECD
+Romania;ROU;EUR;Non-OECD
+Russian Federation;RUS;RUS;Non-OECD
+Rwanda;RWA;rest;Non-OECD
+Saint Barthelemy;BLM;rest;Non-OECD
+Saint Helena, Ascension and Tristan da Cunha;SHN;rest;Non-OECD
+Saint Kitts and Nevis;KNA;rest;Non-OECD
+Saint Lucia;LCA;rest;Non-OECD
+Saint Martin (French part);MAF;rest;Non-OECD
+Saint Pierre and Miquelon;SPM;rest;Non-OECD
+Saint Vincent and the Grenadines;VCT;rest;Non-OECD
+Samoa;WSM;rest;Non-OECD
+San Marino;SMR;rest;Non-OECD
+Sao Tome and Principe;STP;rest;Non-OECD
+Saudi Arabia;SAU;rest;Non-OECD
+Senegal;SEN;rest;Non-OECD
+Serbia;SRB;rest;Non-OECD
+Seychelles;SYC;rest;Non-OECD
+Sierra Leone;SLE;rest;Non-OECD
+Singapore;SGP;ASEAN;Non-OECD
+Sint Maarten (Dutch part);SXM;rest;Non-OECD
+Slovakia;SVK;EUR;OECD
+Slovenia;SVN;EUR;OECD
+Solomon Islands;SLB;rest;Non-OECD
+Somalia;SOM;rest;Non-OECD
+South Africa;ZAF;ZAF;Non-OECD
+South Georgia and the South Sandwich Islands;SGS;rest;Non-OECD
+South Sudan;SSD;rest;Non-OECD
+Spain;ESP;EUR;OECD
+Sri Lanka;LKA;rest;Non-OECD
+Sudan;SDN;rest;Non-OECD
+Suriname;SUR;rest;Non-OECD
+Svalbard and Jan Mayen;SJM;rest;Non-OECD
+Swaziland;SWZ;rest;Non-OECD
+Sweden;SWE;EUR;OECD
+Switzerland;CHE;rest;OECD
+Syrian Arab Republic;SYR;rest;Non-OECD
+Taiwan, Province of China;TWN;rest;Non-OECD
+Tajikistan;TJK;rest;Non-OECD
+Tanzania, United Republic of;TZA;rest;Non-OECD
+Thailand;THA;ASEAN;Non-OECD
+Timor-Leste;TLS;rest;Non-OECD
+Togo;TGO;rest;Non-OECD
+Tokelau;TKL;rest;Non-OECD
+Tonga;TON;rest;Non-OECD
+Trinidad and Tobago;TTO;rest;Non-OECD
+Tunisia;TUN;rest;Non-OECD
+Turkey;TUR;rest;OECD
+Turkmenistan;TKM;rest;Non-OECD
+Turks and Caicos Islands;TCA;rest;Non-OECD
+Tuvalu;TUV;rest;Non-OECD
+Uganda;UGA;rest;Non-OECD
+Ukraine;UKR;rest;Non-OECD
+United Arab Emirates;ARE;rest;Non-OECD
+United Kingdom;GBR;EUR;OECD
+United States;USA;USA;OECD
+United States Minor Outlying Islands;UMI;rest;Non-OECD
+Uruguay;URY;rest;Non-OECD
+Uzbekistan;UZB;rest;Non-OECD
+Vanuatu;VUT;rest;Non-OECD
+Venezuela, Bolivarian Republic of;VEN;rest;Non-OECD
+Viet Nam;VNM;ASEAN;Non-OECD
+Virgin Islands, British;VGB;rest;Non-OECD
+Virgin Islands, U.S.;VIR;rest;Non-OECD
+Wallis and Futuna;WLF;rest;Non-OECD
+Western Sahara;ESH;rest;Non-OECD
+Yemen;YEM;rest;Non-OECD
+Zambia;ZMB;rest;Non-OECD
+Zimbabwe;ZWE;rest;Non-OECD

From ad63420edb1ee44862817572e27f8e82c3c6068c Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Thu, 26 Sep 2024 13:50:10 +0200
Subject: [PATCH 4/8] update reading in AGEB electricity data

---
 R/calcAGEB.R                                  |  13 +-
 R/readAGEB.R                                  |  14 +-
 .../Mapping_AGEB_REMIND.csv                   | 185 ++++++++++++++++++
 3 files changed, 201 insertions(+), 11 deletions(-)
 create mode 100755 inst/extdata/reportingVariables/Mapping_AGEB_REMIND.csv

diff --git a/R/calcAGEB.R b/R/calcAGEB.R
index c3dd65ff..db80c714 100644
--- a/R/calcAGEB.R
+++ b/R/calcAGEB.R
@@ -6,18 +6,17 @@
 #' @author Falk Benke
 #'
 #' @param subtype data subtype. Either "balances" ("Auswertungstabellen zur Energiebilanz Deutschland") or
-#'        "electricity" ("Bruttostromerzeugung in Deutschland nach Energieträgern")
+#'        "electricity" ("Nettostromerzeugung in Deutschland nach Energieträgern")
 #' @importFrom dplyr select mutate left_join
-#' @importFrom rlang sym
 #' @importFrom stats aggregate
 #' @export
 calcAGEB <- function(subtype = "balances") {
   ageb <- readSource("AGEB", subtype = subtype)
 
-  mapping <- toolGetMapping("Mapping_AGEB_REMIND.csv", type = "reportingVariables", where = "mappingfolder") %>%
-    mutate("conversion" = as.numeric(!!sym("Factor")) * !!sym("Weight")) %>%
+  mapping <- toolGetMapping("Mapping_AGEB_REMIND.csv", type = "reportingVariables", where = "mrremind") %>%
+    mutate("conversion" = as.numeric(.data$Factor)) %>%
     select("variable" = "AGEB_variable", "REMIND_variable", "conversion", "unit" = "Unit_AGEB", "Unit_REMIND") %>%
-    filter(!!sym("REMIND_variable") != "")
+    filter(.data$REMIND_variable != "")
 
   x <- left_join(
     ageb %>%
@@ -32,8 +31,8 @@ calcAGEB <- function(subtype = "balances") {
     by = "variable"
   ) %>%
     mutate(
-      "value" = !!sym("value") * !!sym("conversion"),
-      "REMIND_variable" = paste0(!!sym("REMIND_variable"), " (", !!sym("Unit_REMIND"), ")")
+      "value" = .data$value * .data$conversion,
+      "REMIND_variable" = paste0(.data$REMIND_variable, " (", .data$Unit_REMIND, ")")
     ) %>%
     select("variable" = "REMIND_variable", "region", "year", "value")
 
diff --git a/R/readAGEB.R b/R/readAGEB.R
index 57f78e9c..9d542466 100644
--- a/R/readAGEB.R
+++ b/R/readAGEB.R
@@ -82,21 +82,27 @@ readAGEB <- function(subtype = "balances") {
       as.magpie() %>%
       return()
   } else if (subtype == "electricity") {
-    read_xlsx(
+    data <-read_xlsx(
       path = "STRERZ_Abg_02_2024_korr.xlsx",
       sheet = "STRERZ (netto)", col_names = TRUE,
       col_types = c("text", rep("numeric", 34)),
-      range = "B3:AJ17", .name_repair = "minimal", na = c("k.A.")
+      range = "B3:AJ23", .name_repair = "minimal", na = c("k.A.")
     ) %>%
       mutate("TWh" = gsub(", darunter:", "", !!sym("TWh"))) %>%
       mutate("TWh" = gsub("- ", "", !!sym("TWh"))) %>%
-      mutate("TWh" = gsub("[0-9])", "", !!sym("TWh"))) %>%
-      mutate("variable" = paste0("9 Bruttostromerzeugung|", !!sym("TWh"))) %>%
+      mutate("TWh" = gsub("[0-9])", "", !!sym("TWh")))
+
+    data[12,"TWh"] <- "Erneuerbare, darunter Hausmüll"
+    data[16,"TWh"] <- "Sonstige, darunter Hausmüll"
+
+    data <- data %>%
+      mutate("variable" = paste0("9 Nettostromerzeugung|", .data$TWh)) %>%
       select(-1) %>%
       reshape2::melt(id.vars = c("variable"), variable.name = "period", value.name = "value") %>%
       mutate("region" = "DEU", "unit" = "TWh") %>%
       select("region", "period", "variable", "unit", "value") %>%
       as.magpie() %>%
       return()
+
   }
 }
diff --git a/inst/extdata/reportingVariables/Mapping_AGEB_REMIND.csv b/inst/extdata/reportingVariables/Mapping_AGEB_REMIND.csv
new file mode 100755
index 00000000..e7cf2199
--- /dev/null
+++ b/inst/extdata/reportingVariables/Mapping_AGEB_REMIND.csv
@@ -0,0 +1,185 @@
+AGEB_variable;REMIND_variable;Unit_REMIND;Unit_AGEB;Factor;Comment
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Steinkohle;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Braunkohle;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Mineralöle;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Gase;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Erdgas, Erdölgas;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Erneuerbare Energien;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Sonstige Energieträger;;;;;
+1_1 Primaerenergiegewinnung im Inland nach Energietraegern|Insgesamt;;;;;
+1_2 Einfuhr|Steinkohle;Trade|Imports|Coal;EJ/yr;PJ;0.001;
+1_2 Einfuhr|Braunkohle;Trade|Imports|Coal;EJ/yr;PJ;0.001;
+1_2 Einfuhr|Mineralöle;Trade|Imports|Oil;EJ/yr;PJ;0.001;
+1_2 Einfuhr|Gase;Trade|Imports|Gas;EJ/yr;PJ;0.001;
+1_2 Einfuhr|Erdgas, Erdölgas;;;;;
+1_2 Einfuhr|Erneuerbare Energien;;;;;
+1_2 Einfuhr|Sonstige Energieträger;;;;;
+1_2 Einfuhr|Strom;Trade|Imports|SE|Electricity;EJ/yr;PJ;0.001;
+1_2 Einfuhr|Kernenergie;;;;;
+1_2 Einfuhr|Insgesamt;;;;;
+1_3 Ausfuhr|Steinkohle;Trade|Exports|Coal;EJ/yr;PJ;0.001;
+1_3 Ausfuhr|Braunkohle;Trade|Exports|Coal;EJ/yr;PJ;0.001;
+1_3 Ausfuhr|Mineralöle;Trade|Exports|Oil;EJ/yr;PJ;0.001;
+1_3 Ausfuhr|Gase;Trade|Exports|Gas;EJ/yr;PJ;0.001;
+1_3 Ausfuhr|Erdgas, Erdölgas;;;;;
+1_3 Ausfuhr|Erneuerbare Energien;;;;;
+1_3 Ausfuhr|Sonstige Energieträger;;;;;
+1_3 Ausfuhr|Strom;Trade|Exports|SE|Electricity;EJ/yr;PJ;0.001;
+1_3 Ausfuhr|Kernenergie;;;;;
+1_3 Ausfuhr|Insgesamt;;;;;
+1_4 Nettoeinfuhr|Steinkohle;Trade|Coal;EJ/yr;PJ;-0.001;
+1_4 Nettoeinfuhr|Braunkohle;Trade|Coal;EJ/yr;PJ;-0.001;
+1_4 Nettoeinfuhr|Mineralöle;Trade|Oil;EJ/yr;PJ;-0.001;
+1_4 Nettoeinfuhr|Gase;Trade|Gas;EJ/yr;PJ;-0.001;
+1_4 Nettoeinfuhr|Erdgas, Erdölgas;;;;;
+1_4 Nettoeinfuhr|Erneuerbare Energien;;;;;
+1_4 Nettoeinfuhr|Sonstige Energieträger;;;;;
+1_4 Nettoeinfuhr|Strom;Trade|SE|Electricity;EJ/yr;PJ;0.001;
+1_4 Nettoeinfuhr|Kernenergie;;;;;
+1_4 Nettoeinfuhr|Insgesamt;;;;;
+1_5 Bunkerungen seegehender Schiffe|Dieselkraftstoff;;;;;
+1_5 Bunkerungen seegehender Schiffe|Heizöl, leicht;;;;;
+1_5 Bunkerungen seegehender Schiffe|Heizöl, schwer;;;;;
+1_5 Bunkerungen seegehender Schiffe|Andere Mineralölprodukte;;;;;
+1_5 Bunkerungen seegehender Schiffe|Insgesamt;;;;;
+2_1 Primaerenergieverbrauch nach Energietraegern|Steinkohle;PE|Coal;EJ/yr;PJ;0.001;
+2_1 Primaerenergieverbrauch nach Energietraegern|Braunkohle;PE|Coal;EJ/yr;PJ;0.001;
+2_1 Primaerenergieverbrauch nach Energietraegern|Mineralöle;PE|Oil;EJ/yr;PJ;0.001;
+2_1 Primaerenergieverbrauch nach Energietraegern|Gase;PE|Gas;EJ/yr;PJ;0.001;
+2_1 Primaerenergieverbrauch nach Energietraegern|Erdgas, Erdölgas;;;;;
+2_1 Primaerenergieverbrauch nach Energietraegern|Erneuerbare Energien;;;;;
+2_1 Primaerenergieverbrauch nach Energietraegern|Sonstige Energieträger;;;;;
+2_1 Primaerenergieverbrauch nach Energietraegern|Außenhandelssaldo Strom;;;;;
+2_1 Primaerenergieverbrauch nach Energietraegern|Kernenergie;PE|Nuclear;EJ/yr;PJ;0.001;
+2_1 Primaerenergieverbrauch nach Energietraegern|Insgesamt;PE;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Gewinnung im Inland;;;;;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Primärenergieverbrauch;;;;;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Umwandlungseinsatz;;;;;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Umwandlungsausstoß;;;;;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Sonst_ E_-Verbr_, Verluste;;;;;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Nicht-energetischer Verbrauch;FE|Non-energy Use|Industry;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Endenergieverbrauch;FE|w/o Non-energy Use w/o Bunkers;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Bergbau, Verarb_ Gewerbe;FE|w/o Non-energy Use|Industry;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Verkehr;FE|Transport;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Private Haushalte;FE|Buildings;EJ/yr;PJ;0.001;
+2_2 Struktur des Energieverbrauchs nach Sektoren|Gewerbe, Handel, Dienstl_;FE|Buildings;EJ/yr;PJ;0.001;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Wasserkraft;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Windenergie;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Photovoltaik;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Solarthermie;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Geothermie;SE|Heat|Geothermal;EJ/yr;PJ;0.001;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Umweltwärme;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Biomasse;SE|Biomass;EJ/yr;PJ;0.001;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Biogene Abfälle;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Flüssige Biokraftstoffe;;;;;
+3_1 Primaerenergieverbrauch erneuerbare Energien|Insgesamt;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Steinkohle;PE|Coal|Electricity;EJ/yr;PJ;0.001;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Braunkohle;PE|Coal|Electricity;EJ/yr;PJ;0.001;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Mineralöle;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Gase;PE|Gas|Electricity;EJ/yr;PJ;0.001;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Erdgas, Erdölgas;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Erneuerbare Energien;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Sonstige Energieträger;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Strom;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Kernenergie;;;;;
+4_1 Einsatz von Energietraegern zur Stromerzeugung|Insgesamt;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Steinkohle;PE|Coal|Heat;EJ/yr;PJ;0.001;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Braunkohle;PE|Coal|Heat;EJ/yr;PJ;0.001;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Mineralöle;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Gase;PE|Gas|Heat;EJ/yr;PJ;0.001;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Erdgas, Erdölgas;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Erneuerbare Energien;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Sonstige Energieträger;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Strom;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Kernenergie;;;;;
+4_2 Einsatz von Energietraegern zur Fernwaermeerzeugung|Insgesamt;;;;;
+6_1 Endenergieverbrauch nach Energieträgern|Steinkohle;FE|Solids|Fossil|Coal;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Braunkohle;FE|Solids|Fossil|Coal;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Mineralöle;FE|Liquids|Fossil|Oil;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Gase;FE|Gases|Fossil;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_1 Endenergieverbrauch nach Energietraegern|Erneuerbare Energien;;;;;
+6_1 Endenergieverbrauch nach Energietraegern|Sonstige Energieträger;;;;;
+6_1 Endenergieverbrauch nach Energietraegern|Strom;FE|Electricity;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Fernwärme;FE|Heat;EJ/yr;PJ;0.001;
+6_1 Endenergieverbrauch nach Energietraegern|Insgesamt;FE;EJ/yr;PJ;0.001;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Steinkohle;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Braunkohle;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Mineralöle;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Gase;FE|w/o Non-energy Use|Industry|Gases;EJ/yr;PJ;0.001;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Erneuerbare Energien;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Sonstige Energieträger;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Strom;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Fernwärme;;;;;
+6_2 Endenergieverbrauch Bergbau, Gewinnung von Steinen und Erden und Verarbeitendes Gewerbe nach Energietraegern|Insgesamt;;;;;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Steinkohle;FE|Buildings|Solids|Coal;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Braunkohle;FE|Buildings|Solids|Coal;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Mineralöle;FE|Buildings|Liquids|Fossil|Oil;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Gase;FE|Buildings|Gases;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Erneuerbare Energien;;;;;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Sonstige Energieträger;;;;;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Strom;FE|Buildings|Electricity;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Fernwärme;FE|Buildings|Heat;EJ/yr;PJ;0.001;
+6_3 Endenergieverbrauch Private Haushalte nach Energietraegern|Insgesamt;;;;;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Steinkohle;FE|Buildings|Solids|Coal;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Braunkohle;FE|Buildings|Solids|Coal;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Mineralöle;FE|Buildings|Liquids|Fossil|Oil;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Gase;FE|Buildings|Gases;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Erneuerbare Energien;;;;;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Sonstige Energieträger;;;;;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Strom;FE|Buildings|Electricity;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Fernwärme;FE|Buildings|Heat;EJ/yr;PJ;0.001;
+6_4 Endenergieverbrauch Gewerbe, Handel, Dienstleistungen (GHD) nach Energietraegern|Insgesamt;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Ottokraftstoffe;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Dieselkraftstoff;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Heizöl, leicht;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Flüssiggas;FE|w/o Non-energy Use|Industry|Gases;EJ/yr;PJ;0.001;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Erdgas, Erdölgas;FE|w/o Non-energy Use|Industry|Gases;EJ/yr;PJ;0.001;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Biodiesel;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Bioethanol;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Sonstige Biomasse;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Strom;;;;;
+6_5 Endenergieverbrauch Landwirtschaft, Fischerei, Bauwirtschaft nach Energietraegern|Insgesamt;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Steinkohle;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Braunkohle;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Mineralöle;FE|Transport|Liquids|Fossil|Oil;EJ/yr;PJ;0.001;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Gase;FE|Transport|Gases|Fossil|Natural Gas;EJ/yr;PJ;0.001;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Erneuerbare Energien;FE|Transport|Liquids|Biomass;EJ/yr;PJ;0.001;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Sonstige Energieträger;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Strom;FE|Transport|Electricity;EJ/yr;PJ;0.001;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Fernwärme;;;;;
+6_6 Endenergieverbrauch Verkehr nach Energietraegern|Insgesamt;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Ottokraftstoffe;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Dieselkraftstoff;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Flüssiggas;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Erdgas, Erdölgas;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Erneuerbare Energien;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Strom;;;;;
+6_8 Endenergieverbrauch im Subsektor Strassenverkehr nach Energietraegern|Insgesamt;;;;;
+9 Nettostromerzeugung|Braunkohle;SE|Electricity|Coal;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Steinkohle;SE|Electricity|Coal;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Kernenergie;SE|Electricity|Nuclear;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Erdgas;SE|Electricity|Gas;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Mineralöl;SE|Electricity|Oil;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Erneuerbare Energien (EE);;;;;
+9 Nettostromerzeugung|Wind onshore;SE|Electricity|Wind|Onshore;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Wind offshore;SE|Electricity|Wind|Offshore;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Wind onshore;SE|Electricity|Wind;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Wind offshore;SE|Electricity|Wind;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Wasserkraft;SE|Electricity|Hydro;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Biomasse;SE|Electricity|Biomass;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Photovoltaik;SE|Electricity|Solar|PV;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Photovoltaik;SE|Electricity|Solar;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Hausmüll;;;;;
+9 Nettostromerzeugung|Geothermie;SE|Electricity|Geothermal;EJ/yr;TWh;0.0036;
+9 Nettostromerzeugung|Sonstige;;;;;
+9 Nettostromerzeugung|Pumpspeicher;;;;;
+9 Nettostromerzeugung|Sonstige, darunter Hausmüll;;;;;
+9 Nettostromerzeugung|Industrieabfall;;;;;
+9 Nettostromerzeugung|Nettostromerzeugung  inkl_ PSE;;;;;
+9 Nettostromerzeugung|Nettostromerzeugung exkl_ PSE;;;;;
+9 Nettostromerzeugung|Anteil EE an der Nettostromerzeugung [%];;;;;

From 1809421b22d9abe826f9cc8b0fd9ec37daab91db Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Fri, 27 Sep 2024 11:40:38 +0200
Subject: [PATCH 5/8] minor correction

---
 R/calcAGEB.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/calcAGEB.R b/R/calcAGEB.R
index 98e35a92..b6f95011 100644
--- a/R/calcAGEB.R
+++ b/R/calcAGEB.R
@@ -6,7 +6,7 @@
 #' @author Falk Benke
 #'
 #' @param subtype data subtype. Either "balances" ("Auswertungstabellen zur Energiebilanz Deutschland") or
-#'        "electricity" ("Nettostromerzeugung in Deutschland nach Energieträgern")
+#'        "electricity" ("Bruttostromerzeugung in Deutschland nach Energieträgern")
 #' @importFrom dplyr select mutate left_join
 #' @importFrom stats aggregate
 #' @export

From caf890e0bbb1399b8da2555ce5cf84ca0b01fada Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Tue, 1 Oct 2024 15:53:48 +0200
Subject: [PATCH 6/8] refactor calcIEA_ETP

---
 R/calcIEA_ETP.R | 118 ++++++++++++++++++++++++------------------------
 1 file changed, 59 insertions(+), 59 deletions(-)

diff --git a/R/calcIEA_ETP.R b/R/calcIEA_ETP.R
index c3f4ec4d..965b792e 100644
--- a/R/calcIEA_ETP.R
+++ b/R/calcIEA_ETP.R
@@ -6,73 +6,73 @@
 #' @author Falk Benke
 #'
 #' @importFrom dplyr select mutate left_join
-#' @importFrom rlang sym
-#' @importFrom stats aggregate na.pass
 #' @export
 
 calcIEA_ETP <- function() {
+
+  .map <- function(x, mapping) {
+
+    data <- as.data.frame(x) %>%
+      as_tibble() %>%
+      select(
+        "region" = "Region", "scenario" = "Data1", "variable" = "Data2",
+        "year" = "Year", "value" = "Value"
+      ) %>%
+      filter(!is.na(.data$value))
+
+    data <- full_join(data, mapping, by = "variable", relationship = "many-to-many") %>%
+      filter(.data$REMIND != "")
+
+    # composite REMIND variables that must be removed from the data,
+    # because we do not have all the components in the IEA data
+    remove <- data %>% filter(is.na(value))
+
+    data <- data %>%
+      filter(!is.na(.data$value),
+             !(.data$variable %in% unique(remove$REMIND))) %>%
+      mutate(
+        "value" = .data$value * .data$Conversion,
+        "REMIND" = paste0(.data$REMIND, " (", .data$Unit_REMIND, ")"),
+        "model" = paste0("IEA ETP ", .data$scenario),
+        "year" = as.numeric(as.character(.data$year))
+      ) %>%
+      select("region", "year", "model", "variable" = "REMIND", "value")
+
+    x <- aggregate(value ~ region + year + model + variable, data, sum) %>%
+      as.magpie()
+
+    return(x)
+  }
+
   mapping <- toolGetMapping("Mapping_IEA_ETP.csv", type = "reportingVariables", where = "mrremind") %>%
     filter(!is.na(.data$REMIND), .data$REMIND != "") %>%
-    mutate("Conversion" = as.numeric(.data$Conversion)) %>%
-    select("variable" = "IEA_ETP", "REMIND", "Conversion", "Unit_REMIND")
-
-  mapping$variable <- trimws(mapping$variable)
-  mapping$REMIND <- trimws(mapping$REMIND)
-
-  x1 <- readSource("IEA_ETP", subtype = "industry")
-  x2 <- readSource("IEA_ETP", subtype = "transport")
-  x3 <- readSource("IEA_ETP", subtype = "buildings")
-  x4 <- readSource("IEA_ETP", subtype = "summary")
-
-  x5 <- mbind(
-    readSource("IEA_ETP", subtype = "industry", convert = F)["WORLD", , ],
-    readSource("IEA_ETP", subtype = "transport", convert = F)["WORLD", , ],
-    readSource("IEA_ETP", subtype = "buildings", convert = F)["WORLD", , ],
-    readSource("IEA_ETP", subtype = "summary", convert = F)["WORLD", , ]
+    mutate(
+      "Conversion" = as.numeric(.data$Conversion),
+      "variable" = trimws(.data$IEA_ETP),
+      "REMIND" = trimws(.data$REMIND)
+      ) %>%
+    select("variable", "REMIND", "Conversion", "Unit_REMIND")
+
+
+  xReg <- mbind(
+    readSource("IEA_ETP", subtype = "industry"),
+    readSource("IEA_ETP", subtype = "transport"),
+    readSource("IEA_ETP", subtype = "buildings"),
+    readSource("IEA_ETP", subtype = "summary")
   )
 
-
-  data <- as.data.frame(mbind(x1, x2, x3, x4)) %>%
-    as_tibble() %>%
-    select(
-      "region" = "Region", "scenario" = "Data1", "variable" = "Data2",
-      "year" = "Year", "value" = "Value"
-    ) %>%
-    filter(!is.na(.data$value))
-
-  dataGlo <- as.data.frame(x5) %>%
-    as_tibble() %>%
-    select(
-      "region" = "Region", "scenario" = "Data1", "variable" = "Data2",
-      "year" = "Year", "value" = "Value"
-    ) %>%
-    filter(!is.na(.data$value)) %>%
-    mutate(region = "GLO")
-
-
-
-  x <- left_join(
-    rbind(data, dataGlo),
-    mapping,
-    by = "variable",
-    relationship = "many-to-many"
-  ) %>%
-    filter(.data$REMIND != "") %>%
-    mutate(
-      "value" = .data$value * .data$Conversion,
-      "REMIND" = paste0(.data$REMIND, " (", .data$Unit_REMIND, ")"),
-      "model" = paste0("IEA ETP ", .data$scenario),
-      "year" = as.numeric(as.character(.data$year))
-    ) %>%
-    select("region", "year", "model", "variable" = "REMIND", "value")
-
-  x <- aggregate(value ~ region + year + model + variable, x, sum) %>%
-    as.magpie()
-
-  dataGlo <- x["GLO", , ]
-  x <- x["GLO", , invert = T] %>%
+  dataReg <- .map(xReg, mapping) %>%
     toolCountryFill(fill = NA, verbosity = 2)
 
+  xGlo <- mbind(
+    readSource("IEA_ETP", subtype = "industry", convert = FALSE)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "transport", convert = FALSE)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "buildings", convert = FALSE)["WORLD", , ],
+    readSource("IEA_ETP", subtype = "summary", convert = FALSE)["WORLD", , ]
+  )
+
+  getItems(xGlo, dim = 1) <- "GLO"
+  dataGlo <- .map(xGlo, mapping)
 
   # includes global values from the original source instead of calculating
   # them as the sum of all countries (as countries are incomplete)
@@ -100,7 +100,7 @@ calcIEA_ETP <- function() {
   }
 
   return(list(
-    x = x,
+    x = dataReg,
     weight = NULL,
     aggregationFunction = .customAggregate,
     aggregationArguments = list(glo = dataGlo),

From b844e2b454ea1d09ef1652312aabb47d83efe40e Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Tue, 1 Oct 2024 17:27:18 +0200
Subject: [PATCH 7/8] fix bug in calcIEA_ETP

---
 R/calcIEA_ETP.R | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/R/calcIEA_ETP.R b/R/calcIEA_ETP.R
index 965b792e..91b7834c 100644
--- a/R/calcIEA_ETP.R
+++ b/R/calcIEA_ETP.R
@@ -29,7 +29,7 @@ calcIEA_ETP <- function() {
 
     data <- data %>%
       filter(!is.na(.data$value),
-             !(.data$variable %in% unique(remove$REMIND))) %>%
+             !(.data$REMIND %in% unique(remove$REMIND))) %>%
       mutate(
         "value" = .data$value * .data$Conversion,
         "REMIND" = paste0(.data$REMIND, " (", .data$Unit_REMIND, ")"),
@@ -53,7 +53,6 @@ calcIEA_ETP <- function() {
       ) %>%
     select("variable", "REMIND", "Conversion", "Unit_REMIND")
 
-
   xReg <- mbind(
     readSource("IEA_ETP", subtype = "industry"),
     readSource("IEA_ETP", subtype = "transport"),
@@ -64,6 +63,9 @@ calcIEA_ETP <- function() {
   dataReg <- .map(xReg, mapping) %>%
     toolCountryFill(fill = NA, verbosity = 2)
 
+  # set 0s in other CHA countries than China to approximate CHA as China
+  dataReg[c("HKG", "MAC", "TWN"), , ] <- 0
+
   xGlo <- mbind(
     readSource("IEA_ETP", subtype = "industry", convert = FALSE)["WORLD", , ],
     readSource("IEA_ETP", subtype = "transport", convert = FALSE)["WORLD", , ],
@@ -99,6 +101,9 @@ calcIEA_ETP <- function() {
     }
   }
 
+
+
+
   return(list(
     x = dataReg,
     weight = NULL,

From cfb3fa22c6e2de93146fa9de7798c58893a167d8 Mon Sep 17 00:00:00 2001
From: Falk Benke <falk.benke@pik-potsdam.de>
Date: Tue, 1 Oct 2024 17:48:26 +0200
Subject: [PATCH 8/8] increment version

---
 .buildlibrary                    | 2 +-
 CITATION.cff                     | 4 ++--
 DESCRIPTION                      | 4 ++--
 NAMESPACE                        | 1 -
 R/calcIEA_ETP.R                  | 4 ++--
 R/readEuropeanEnergyDatasheets.R | 2 +-
 README.md                        | 6 +++---
 7 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/.buildlibrary b/.buildlibrary
index 79a973de..c227e38a 100644
--- a/.buildlibrary
+++ b/.buildlibrary
@@ -1,4 +1,4 @@
-ValidationKey: '38572284'
+ValidationKey: '384142370'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'
diff --git a/CITATION.cff b/CITATION.cff
index 8a7b6240..ec5a8fd8 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'mrremind: MadRat REMIND Input Data Package'
-version: 0.192.9
-date-released: '2024-09-30'
+version: 0.192.10
+date-released: '2024-10-01'
 abstract: The mrremind packages contains data preprocessing for the REMIND model.
 authors:
 - family-names: Baumstark
diff --git a/DESCRIPTION b/DESCRIPTION
index 7ef934df..8c1e6c09 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,8 +1,8 @@
 Type: Package
 Package: mrremind
 Title: MadRat REMIND Input Data Package
-Version: 0.192.9
-Date: 2024-09-30
+Version: 0.192.10
+Date: 2024-10-01
 Authors@R: c(
     person("Lavinia", "Baumstark", , "lavinia@pik-potsdam.de", role = c("aut", "cre")),
     person("Renato", "Rodrigues", role = "aut"),
diff --git a/NAMESPACE b/NAMESPACE
index 04590f83..7635260f 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -47,7 +47,6 @@ export(convertUBA)
 export(convertUNFCCC)
 export(convertUNIDO)
 export(convertUSGS)
-export(filter_historical_mif)
 export(fullTHRESHOLDS)
 export(readADVANCE_WP2)
 export(readAGEB)
diff --git a/R/calcIEA_ETP.R b/R/calcIEA_ETP.R
index 91b7834c..ed8ce566 100644
--- a/R/calcIEA_ETP.R
+++ b/R/calcIEA_ETP.R
@@ -25,7 +25,7 @@ calcIEA_ETP <- function() {
 
     # composite REMIND variables that must be removed from the data,
     # because we do not have all the components in the IEA data
-    remove <- data %>% filter(is.na(value))
+    remove <- data %>% filter(is.na(.data$value))
 
     data <- data %>%
       filter(!is.na(.data$value),
@@ -50,7 +50,7 @@ calcIEA_ETP <- function() {
       "Conversion" = as.numeric(.data$Conversion),
       "variable" = trimws(.data$IEA_ETP),
       "REMIND" = trimws(.data$REMIND)
-      ) %>%
+    ) %>%
     select("variable", "REMIND", "Conversion", "Unit_REMIND")
 
   xReg <- mbind(
diff --git a/R/readEuropeanEnergyDatasheets.R b/R/readEuropeanEnergyDatasheets.R
index 5fa229bf..3bb72574 100644
--- a/R/readEuropeanEnergyDatasheets.R
+++ b/R/readEuropeanEnergyDatasheets.R
@@ -15,7 +15,7 @@
 #' or "EU27" (latest data from August 23 without GBR)
 #' @importFrom tidyr drop_na extract
 #' @importFrom readxl excel_sheets read_excel
-#' @importFrom stats aggregate
+#' @importFrom stats aggregate na.pass
 #'
 readEuropeanEnergyDatasheets <- function(subtype) {
   if (!subtype %in% c("EU27", "EU28")) {
diff --git a/README.md b/README.md
index 8b74075b..e73e0452 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # MadRat REMIND Input Data Package
 
-R package **mrremind**, version **0.192.9**
+R package **mrremind**, version **0.192.10**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/mrremind)](https://cran.r-project.org/package=mrremind)  [![R build status](https://github.com/pik-piam/mrremind/workflows/check/badge.svg)](https://github.com/pik-piam/mrremind/actions) [![codecov](https://codecov.io/gh/pik-piam/mrremind/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrremind) [![r-universe](https://pik-piam.r-universe.dev/badges/mrremind)](https://pik-piam.r-universe.dev/builds)
 
@@ -39,7 +39,7 @@ In case of questions / problems please contact Lavinia Baumstark <lavinia@pik-po
 
 To cite package **mrremind** in publications use:
 
-Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.192.9, <https://github.com/pik-piam/mrremind>.
+Baumstark L, Rodrigues R, Levesque A, Oeser J, Bertram C, Mouratiadou I, Malik A, Schreyer F, Soergel B, Rottoli M, Mishra A, Dirnaichner A, Pehl M, Giannousakis A, Klein D, Strefler J, Feldhaus L, Brecha R, Rauner S, Dietrich J, Bi S, Benke F, Weigmann P, Richters O, Hasse R, Fuchs S, Mandaroux R, Koch J (2024). _mrremind: MadRat REMIND Input Data Package_. R package version 0.192.10, <https://github.com/pik-piam/mrremind>.
 
 A BibTeX entry for LaTeX users is
 
@@ -48,7 +48,7 @@ A BibTeX entry for LaTeX users is
   title = {mrremind: MadRat REMIND Input Data Package},
   author = {Lavinia Baumstark and Renato Rodrigues and Antoine Levesque and Julian Oeser and Christoph Bertram and Ioanna Mouratiadou and Aman Malik and Felix Schreyer and Bjoern Soergel and Marianna Rottoli and Abhijeet Mishra and Alois Dirnaichner and Michaja Pehl and Anastasis Giannousakis and David Klein and Jessica Strefler and Lukas Feldhaus and Regina Brecha and Sebastian Rauner and Jan Philipp Dietrich and Stephen Bi and Falk Benke and Pascal Weigmann and Oliver Richters and Robin Hasse and Sophie Fuchs and Rahel Mandaroux and Johannes Koch},
   year = {2024},
-  note = {R package version 0.192.9},
+  note = {R package version 0.192.10},
   url = {https://github.com/pik-piam/mrremind},
 }
 ```