From d591e88d4b608a488ec46f232f0bc8c8433d23ec Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Thu, 22 Aug 2024 16:43:31 +0200
Subject: [PATCH 1/7] varHarm

---
 .buildlibrary              |   2 +-
 .pre-commit-config.yaml    |   2 +-
 CITATION.cff               |   4 +-
 DESCRIPTION                |   7 +--
 NAMESPACE                  |   3 ++
 R/harmonizeOutput.R        | 105 +++++++++++++++++++++++++++++++++++++
 R/reportEdgeTransport.R    |  11 +++-
 R/storeData.R              |   6 ++-
 README.md                  |   6 +--
 man/harmonizeOutput.Rd     |  27 ++++++++++
 man/reportEdgeTransport.Rd |   3 ++
 11 files changed, 163 insertions(+), 13 deletions(-)
 create mode 100644 R/harmonizeOutput.R
 create mode 100644 man/harmonizeOutput.Rd

diff --git a/.buildlibrary b/.buildlibrary
index ec48bea..44310b1 100644
--- a/.buildlibrary
+++ b/.buildlibrary
@@ -1,4 +1,4 @@
-ValidationKey: '239208'
+ValidationKey: '199570'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 62f13da..7ccf77e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
     -   id: mixed-line-ending
 
 -   repo: https://github.com/lorenzwalthert/precommit
-    rev: 7910e0323d7213f34275a7a562b9ef0fde8ce1b9  # frozen: v0.4.2
+    rev: bae853d82da476eee0e0a57960ee6b741a3b3fb7  # frozen: v0.4.3
     hooks:
     -   id: parsable-R
     -   id: deps-in-desc
diff --git a/CITATION.cff b/CITATION.cff
index 393ab15..1d4b9b9 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'reporttransport: Reporting package for edgeTransport'
-version: 0.0.12
-date-released: '2024-07-30'
+version: 0.1.0
+date-released: '2024-08-22'
 abstract: This package contains edgeTransport-specific routines to report model results.
   The main functionality is to generate transport reporting variables in MIF format
   from a given edgeTransport model run folder or REMIND input data.
diff --git a/DESCRIPTION b/DESCRIPTION
index 6754f4f..e1401be 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,8 +1,8 @@
 Type: Package
 Package: reporttransport
 Title: Reporting package for edgeTransport
-Version: 0.0.12
-Date: 2024-07-30
+Version: 0.1.0
+Date: 2024-08-22
 Authors@R:
     person("Johanna", "Hoppe", , "johanna.hoppe@pik-potsdam.de", role = c("aut", "cre"))
 Description: This package contains edgeTransport-specific routines to
@@ -21,7 +21,8 @@ Imports:
     quitte,
     remind2,
     rmndt,
-    utils
+    utils, 
+    tibble
 Suggests:
     knitr,
     sf,
diff --git a/NAMESPACE b/NAMESPACE
index 934bad8..1e168dc 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -3,6 +3,7 @@
 export(aggregateVariables)
 export(checkForNAsAndDups)
 export(convertToMIF)
+export(harmonizeOutput)
 export(prepareForREMIND)
 export(reportAggregatedCosts)
 export(reportAnalyticsVarSet)
@@ -31,10 +32,12 @@ importFrom(gdxrrw,rgdx.param)
 importFrom(gdxrrw,rgdx.scalar)
 importFrom(quitte,aggregate_map)
 importFrom(quitte,as.quitte)
+importFrom(quitte,read.quitte)
 importFrom(quitte,write.mif)
 importFrom(remind2,toolRegionSubsets)
 importFrom(rmndt,approx_dt)
 importFrom(rmndt,disaggregate_dt)
 importFrom(rmndt,magpie2dt)
+importFrom(tibble,tribble)
 importFrom(utils,packageVersion)
 importFrom(utils,write.csv)
diff --git a/R/harmonizeOutput.R b/R/harmonizeOutput.R
new file mode 100644
index 0000000..00f30f7
--- /dev/null
+++ b/R/harmonizeOutput.R
@@ -0,0 +1,105 @@
+#' Harmonize data to last REMIND iteration in coupled mode
+#'
+#' The two models are coupled via energy service demand, energy intensity and capital expenditure on CES node level.
+#' As REMIND converges the output between EDGE-T and REMIND still slightly differs.
+#' This function calculates harmonization factors that are applied on the EDGE-T energy intensity and final energy values.
+#' Furthermore, the harmonization factors indicate how well the two models converged and can be used for analysis purposes.
+#'
+#' @param edgetOutputDir edgeTransport output directory
+#' @param baseVarSet calculated base variable set including energy intensity and final energy on fleet level
+#' @param data list of raw model output data
+#'
+#' @returns The function returns a list with the harmonized energy intensity and final energy data
+#' @author Johanna Hoppe
+#' @importFrom quitte read.quitte
+#' @importFrom tibble tribble
+#' @import data.table
+#' @export
+
+harmonizeOutput <- function(edgetOutputDir, baseVarSet, data) {
+
+    ## Calculate FE harmonization factors that are then applied to the energy intensity
+    REMINDoutput <- as.data.table(read.quitte(data$remindReportingFile))
+    harmMap <- tribble(
+      ~variable,                                                  ~ all_teEs,
+      "FE|Transport|Freight|Short-Medium distance|Electricity",    "te_eselt_frgt_sm",
+      "FE|Transport|Freight|Short-Medium distance|Liquids",        "te_esdie_frgt_sm",
+      "FE|Transport|Freight|Short-Medium distance|Hydrogen",       "te_esh2t_frgt_sm",
+      "FE|Transport|Freight|Short-Medium distance|Gases",          "te_esgat_frgt_sm",
+      "FE|Transport|Freight|Long distance|Diesel Liquids",         "te_esdie_frgt_lo",
+      "FE|Transport|Pass|Short-Medium distance|Electricity",       "te_eselt_pass_sm",
+      "FE|Transport|Pass|Short-Medium distance|Diesel Liquids",    "te_esdie_pass_sm",
+      "FE|Transport|Pass|Short-Medium distance|Petrol Liquids",    "te_espet_pass_sm",
+      "FE|Transport|Pass|Short-Medium distance|Hydrogen",          "te_esh2t_pass_sm",
+      "FE|Transport|Pass|Short-Medium distance|Gases",             "te_esgat_pass_sm",
+      "FE|Transport|Pass|Long distance|Diesel Liquids",            "te_esdie_pass_lo"
+    )
+    REMINDoutputToHarmonize <- REMINDoutput[variable %in% unique(harmMap$variable)]
+    if (!length(unique(REMINDoutputToHarmonize$variable)) == length(unique(harmMap$variable))) {
+      stop("Variables for EDGE-T harmonization are missing in the REMIND output. Please check reportEdgeTransport()")
+    }
+    demByTechMap <- unique(data$helpers$mapEdgeToREMIND[, c("all_teEs", "univocalName", "technology")])
+    demByTechMap <- demByTechMap[!is.na(all_teEs)]
+    demByTechMap[technology %in% c("BEV", "Electric"), technology := "Electricity"]
+    demByTechMap[technology == "FCEV", technology := "Hydrogen"]
+    EDGEToutputToHarmonize <- merge(baseVarSet$ext$fleetFEdemand[period %in% timeResReporting], demByTechMap, by = c("univocalName", "technology"), all.x = TRUE) # nolint: object_name_linter
+    EDGEToutputToHarmonize <- EDGEToutputToHarmonize[period %in% timeResReporting & !subsectorL1 %in% c("Walk", "Cycle")]
+    EDGEToutputToHarmonize[, variable := NULL][, unit := NULL]
+    EDGEToutputToHarmonize <- merge(EDGEToutputToHarmonize, harmMap, by = "all_teEs", all.x = TRUE)
+    EDGEToutputToHarmonize <- EDGEToutputToHarmonize[, .(value = sum(value)), by = c("region", "period", "variable", "all_teEs")]
+    setnames(EDGEToutputToHarmonize, "value", "edget")
+    setnames(REMINDoutputToHarmonize, "value", "remind")
+    harmFactors <- merge(EDGEToutputToHarmonize, REMINDoutputToHarmonize, by = c("region", "period", "variable"), all.x = TRUE)
+    harmFactors[, factor := ifelse(!remind < 1e-6 & !edget < 1e-6, remind/edget, 1)][, c("edget", "remind") := NULL]
+    harmFactors[, variable := paste0("Harmonization factor|", variable)][, unit := "-"][, scenario:= data$scenarioName][, model:= data$modelName]
+    storeData(edgetOutputDir, harmFactors)
+
+    #Hybrids are listed in energy intensity but are attributed partly to electricity and petrol in the reporting of final energy
+    #The liquids part of the final energy needs to be scaled differntly than the electricity part. This is not possible using the current approach.
+    #Hence, the energy intensity of hybrids is not scaled and the FE is scaled in postprocessing. This leads to a small inconsistency in the dataset
+    #of eneryg service demand, energy intensity and final energy
+
+    #Apply factors
+    harmonizedEnergyIntensity <- merge(baseVarSet$int$fleetEnergyIntensity[period %in% timeResReporting], 
+      data$helpers$mapEdgeToREMIND[, c("all_teEs", "univocalName", "technology")], by = c("univocalName", "technology"), all.x = TRUE)
+    harmonizedEnergyIntensity <- merge(harmonizedEnergyIntensity, harmFactors[, c("region", "period", "all_teEs", "factor")], 
+      by = c("region", "period", "all_teEs"), all.x = TRUE)
+    harmonizedEnergyIntensity[technology == "Hybrid electric", factor := 1]
+    harmonizedEnergyIntensity[, value := ifelse(!subsectorL1 %in% c("Walk", "Cycle"), value * factor, value)][, c("factor", "all_teEs") := NULL]
+    if (anyNA(harmonizedEnergyIntensity)) stop("Variable harmonization did not work. Please check harmonizeOutput() in reporttransport.")
+
+    ## Harmonize final energy
+    harmonizedFinalEnergy <- merge(baseVarSet$ext$fleetFEdemand[period %in% timeResReporting], 
+      demByTechMap, by = c("univocalName", "technology"), all.x = TRUE)
+    harmonizedFinalEnergy <- merge(harmonizedFinalEnergy, harmFactors[, c("region", "period", "all_teEs", "factor")], 
+      by = c("region", "period", "all_teEs"), all.x = TRUE)
+    harmonizedFinalEnergy[, value := ifelse(!subsectorL1 %in% c("Walk", "Cycle"), value * factor, value)]
+    FEcheck <- harmonizedFinalEnergy[!subsectorL1 %in% c("Walk", "Cycle")]
+    harmonizedFinalEnergy[, c("factor", "all_teEs") := NULL]    
+    if (anyNA(harmonizedFinalEnergy)) stop("Variable harmonization did not work. Please check harmonizeOutput() in reporttransport.") 
+    
+    ## Check if harmonization worked and how big the deviation due to the hybrids is
+    # test harmonization
+    FEcheck <- FEcheck[, .(value = sum(value)), by = c("region", "period", "all_teEs")]
+    FEcheck <- merge(FEcheck, harmMap, by = "all_teEs", all.x = TRUE)
+    FEcheck <- merge(FEcheck, REMINDoutputToHarmonize, by = c("region", "period", "variable"), all.x = TRUE)
+    FEcheck[, diff := ifelse(!remind < 1e-6 & !value < 1e-6, (value - remind) / value, 0)]
+    if (max(FEcheck$diff > 1e-5)) stop("FE harmonization did not work. Please check harmonizeOutput()")
+    
+    # test the deviation (as if the dataset would be consistent)
+    calculatedFinalEnergy <- reportFinalEnergy(harmonizedEnergyIntensity, baseVarSet$ext$fleetESdemand, data$scenSpecLoadFactor, data$hybridElecShare, data$helpers)
+    test <- calculatedFinalEnergy[period %in% timeResReporting & !subsectorL1 %in% c("Walk", "Cycle")]
+    test[, variable := NULL][, unit := NULL]
+    test <- merge(test, demByTechMap, by = c("univocalName", "technology"), all.x = TRUE)
+    test <- merge(test, harmMap, by = "all_teEs", all.x = TRUE)
+    test <- test[, .(value = sum(value)), by = c("region", "period", "variable")]
+    test <- merge(test, REMINDoutputToHarmonize, by = c("region", "period", "variable"), all.x = TRUE)
+    test[, diff := ifelse(!remind < 1e-6 & !value < 1e-6, (value - remind) / value, 0)] 
+
+    harmonizedVars <- list(
+      harmonizedEnergyIntensity = harmonizedEnergyIntensity,
+      harmonizedFinalEnergy = harmonizedFinalEnergy
+    )
+
+    return(harmonizedVars)
+}
diff --git a/R/reportEdgeTransport.R b/R/reportEdgeTransport.R
index b11fae0..6d5b299 100644
--- a/R/reportEdgeTransport.R
+++ b/R/reportEdgeTransport.R
@@ -30,17 +30,19 @@
 #' @param isAnalyticsReported Switch for activating reporting of model analytics data
 #' @param isREMINDinputReported Switch for activating reporting of REMIND input data
 #' @param isStored Switch for activating data storage and creating the transport.MIF file
+#' @param isHarmonized Switch for activating ouput harmonization with REMIND
 #'
 #' @returns The function either returns the REMINDinputData if isREMINDinputReported is
 #'          enabled or the transport data in MIF format
 #' @author Johanna Hoppe
 #' @importFrom quitte write.mif
+#' @importFrom tibble tribble
 #' @import data.table
 #' @export
 
 reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NULL, isTransportReported = TRUE,
                                 isTransportExtendedReported = FALSE, isAnalyticsReported = FALSE,
-                                isREMINDinputReported = FALSE, isStored = TRUE, ...) {
+                                isREMINDinputReported = FALSE, isStored = TRUE, isHarmonized = FALSE, ...) {
 
   # If you want to change timeResReporting to timesteps outside the modeleled timesteps,
   # please add an interpolation step
@@ -115,11 +117,18 @@ reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NU
     if (is.null(data$scenSpecPrefTrends)) data$scenSpecPrefTrends <- readRDS(file.path(folderPath, "2_InputDataPolicy", "scenSpecPrefTrends.RDS"))
     if (is.null(data$initialIncoCosts)) data$initialIncoCosts <- readRDS(file.path(folderPath, "2_InputDataPolicy", "initialIncoCosts.RDS"))
   }
+
   #########################################################################
   ## Report output variables
   #########################################################################
   # Base variable set that is needed to report REMIND input data and additional detailed transport data
   baseVarSet <- reportBaseVarSet(data = data, timeResReporting = timeResReporting)
+  if (isHarmonized) {
+    harmonizedVars <- harmonizeOutput(edgetOutputDir, baseVarSet, data)
+    baseVarSet$int$fleetEnergyIntensity <- harmonizedVars$harmonizedEnergyIntensity
+    baseVarSet$ext$fleetFEdemand <- harmonizedVars$harmonizedFinalEnergy
+  }
+  
   reporting <- baseVarSet
   outputVars <- baseVarSet
 
diff --git a/R/storeData.R b/R/storeData.R
index daed1e1..72c58e6 100644
--- a/R/storeData.R
+++ b/R/storeData.R
@@ -44,8 +44,10 @@ storeData <- function(outputFolder, varsList = NULL, ...) {
                        "fleetVehNumbersIterations",
                        "endogenousCostsIterations",
                        "endogenousCosts",
+                       "sectorESdemand",
                        "ESdemandFVsalesLevel",
-                       "fleetVehiclesPerTech")) subfolder <- "4_Output"
+                       "fleetVehiclesPerTech",
+                       "harmFactors")) subfolder <- "4_Output"
     if (varName %in% c("REMINDinputData")) subfolder <- "5_REMINDinputData"
     if (is.null(subfolder)) stop(paste0("No subfolder assigned to ", varName))
 
@@ -59,7 +61,7 @@ storeData <- function(outputFolder, varsList = NULL, ...) {
 
   storeCSV <- function(varName, vars, outputFolder, subfolder = NULL) {
     if (is.null(subfolder)) subfolder <- allocateFile(varName)
-    write.csv(vars[[varName]], file.path(outputFolder, subfolder, paste0(varName, ".csv")))
+    write.csv(vars[[varName]], file.path(outputFolder, subfolder, paste0(varName, ".csv")), row.names = FALSE)
   }
   vars <- list()
   if (!is.null(varsList)) vars <- varsList
diff --git a/README.md b/README.md
index 9270ca8..ba4b32f 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Reporting package for edgeTransport
 
-R package **reporttransport**, version **0.0.12**
+R package **reporttransport**, version **0.1.0**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/reporttransport)](https://cran.r-project.org/package=reporttransport)  [![R build status](https://github.com/pik-piam/reporttransport/workflows/check/badge.svg)](https://github.com/pik-piam/reporttransport/actions) [![codecov](https://codecov.io/gh/pik-piam/reporttransport/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/reporttransport) [![r-universe](https://pik-piam.r-universe.dev/badges/reporttransport)](https://pik-piam.r-universe.dev/builds)
 
@@ -41,7 +41,7 @@ In case of questions / problems please contact Johanna Hoppe <johanna.hoppe@pik-
 
 To cite package **reporttransport** in publications use:
 
-Hoppe J (2024). _reporttransport: Reporting package for edgeTransport_. R package version 0.0.12, <https://github.com/pik-piam/reporttransport>.
+Hoppe J (2024). _reporttransport: Reporting package for edgeTransport_. R package version 0.1.0, <https://github.com/pik-piam/reporttransport>.
 
 A BibTeX entry for LaTeX users is
 
@@ -50,7 +50,7 @@ A BibTeX entry for LaTeX users is
   title = {reporttransport: Reporting package for edgeTransport},
   author = {Johanna Hoppe},
   year = {2024},
-  note = {R package version 0.0.12},
+  note = {R package version 0.1.0},
   url = {https://github.com/pik-piam/reporttransport},
 }
 ```
diff --git a/man/harmonizeOutput.Rd b/man/harmonizeOutput.Rd
new file mode 100644
index 0000000..147b1f5
--- /dev/null
+++ b/man/harmonizeOutput.Rd
@@ -0,0 +1,27 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/harmonizeOutput.R
+\name{harmonizeOutput}
+\alias{harmonizeOutput}
+\title{Harmonize data to last REMIND iteration in coupled mode}
+\usage{
+harmonizeOutput(edgetOutputDir, baseVarSet, data)
+}
+\arguments{
+\item{edgetOutputDir}{edgeTransport output directory}
+
+\item{baseVarSet}{calculated base variable set including energy intensity and final energy on fleet level}
+
+\item{data}{list of raw model output data}
+}
+\value{
+The function returns a list with the harmonized energy intensity and final energy data
+}
+\description{
+The two models are coupled via energy service demand, energy intensity and capital expenditure on CES node level.
+As REMIND converges the output between EDGE-T and REMIND still slightly differs.
+This function calculates harmonization factors that are applied on the EDGE-T energy intensity and final energy values.
+Furthermore, the harmonization factors indicate how well the two models converged and can be used for analysis purposes.
+}
+\author{
+Johanna Hoppe
+}
diff --git a/man/reportEdgeTransport.Rd b/man/reportEdgeTransport.Rd
index 790c129..9099656 100644
--- a/man/reportEdgeTransport.Rd
+++ b/man/reportEdgeTransport.Rd
@@ -12,6 +12,7 @@ reportEdgeTransport(
   isAnalyticsReported = FALSE,
   isREMINDinputReported = FALSE,
   isStored = TRUE,
+  isHarmonized = FALSE,
   ...
 )
 }
@@ -31,6 +32,8 @@ needed to create transportCompareScenarios}
 
 \item{isStored}{Switch for activating data storage and creating the transport.MIF file}
 
+\item{isHarmonized}{Switch for activating ouput harmonization with REMIND}
+
 \item{...}{Additional model results/parameter that are handed over individually}
 }
 \value{

From 2071015b8acff6987b72922c4b04c768dca84290 Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Thu, 22 Aug 2024 17:57:05 +0200
Subject: [PATCH 2/7] remove duplicates

---
 R/harmonizeOutput.R     | 3 +--
 R/reportEdgeTransport.R | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/R/harmonizeOutput.R b/R/harmonizeOutput.R
index 00f30f7..6341926 100644
--- a/R/harmonizeOutput.R
+++ b/R/harmonizeOutput.R
@@ -16,10 +16,9 @@
 #' @import data.table
 #' @export
 
-harmonizeOutput <- function(edgetOutputDir, baseVarSet, data) {
+harmonizeOutput <- function(REMINDoutput, edgetOutputDir, baseVarSet, data) {
 
     ## Calculate FE harmonization factors that are then applied to the energy intensity
-    REMINDoutput <- as.data.table(read.quitte(data$remindReportingFile))
     harmMap <- tribble(
       ~variable,                                                  ~ all_teEs,
       "FE|Transport|Freight|Short-Medium distance|Electricity",    "te_eselt_frgt_sm",
diff --git a/R/reportEdgeTransport.R b/R/reportEdgeTransport.R
index 6d5b299..1413073 100644
--- a/R/reportEdgeTransport.R
+++ b/R/reportEdgeTransport.R
@@ -124,7 +124,8 @@ reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NU
   # Base variable set that is needed to report REMIND input data and additional detailed transport data
   baseVarSet <- reportBaseVarSet(data = data, timeResReporting = timeResReporting)
   if (isHarmonized) {
-    harmonizedVars <- harmonizeOutput(edgetOutputDir, baseVarSet, data)
+    REMINDoutput <- as.data.table(read.quitte(data$remindReportingFile))
+    harmonizedVars <- harmonizeOutput(REMINDoutput, edgetOutputDir, baseVarSet, data)
     baseVarSet$int$fleetEnergyIntensity <- harmonizedVars$harmonizedEnergyIntensity
     baseVarSet$ext$fleetFEdemand <- harmonizedVars$harmonizedFinalEnergy
   }
@@ -167,6 +168,7 @@ reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NU
                               gdx                         = data$gdxPath,
                               isTransportExtendedReported = isTransportExtendedReported)
 
+    if (isHarmonized) reporting <- reporting[!variable %in% REMINDoutput$variable]
     if (isStored) write.mif(reporting, file.path(folderPath, "Transport.mif"))
   }
 

From de027e05c8b7933cf0a4052a9b34c3df69a3d308 Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Thu, 22 Aug 2024 18:27:00 +0200
Subject: [PATCH 3/7] fix storeData call

---
 R/harmonizeOutput.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/harmonizeOutput.R b/R/harmonizeOutput.R
index 6341926..1f3e64c 100644
--- a/R/harmonizeOutput.R
+++ b/R/harmonizeOutput.R
@@ -51,7 +51,7 @@ harmonizeOutput <- function(REMINDoutput, edgetOutputDir, baseVarSet, data) {
     harmFactors <- merge(EDGEToutputToHarmonize, REMINDoutputToHarmonize, by = c("region", "period", "variable"), all.x = TRUE)
     harmFactors[, factor := ifelse(!remind < 1e-6 & !edget < 1e-6, remind/edget, 1)][, c("edget", "remind") := NULL]
     harmFactors[, variable := paste0("Harmonization factor|", variable)][, unit := "-"][, scenario:= data$scenarioName][, model:= data$modelName]
-    storeData(edgetOutputDir, harmFactors)
+    storeData(edgetOutputDir, list(harmFactors = harmFactors))
 
     #Hybrids are listed in energy intensity but are attributed partly to electricity and petrol in the reporting of final energy
     #The liquids part of the final energy needs to be scaled differntly than the electricity part. This is not possible using the current approach.

From b3ebadb539089bfae0723dd30b4d583c17ff0b9a Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Thu, 22 Aug 2024 19:07:51 +0200
Subject: [PATCH 4/7] buildLib

---
 R/harmonizeOutput.R    | 2 +-
 man/harmonizeOutput.Rd | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/R/harmonizeOutput.R b/R/harmonizeOutput.R
index 1f3e64c..ce4aa81 100644
--- a/R/harmonizeOutput.R
+++ b/R/harmonizeOutput.R
@@ -4,7 +4,7 @@
 #' As REMIND converges the output between EDGE-T and REMIND still slightly differs.
 #' This function calculates harmonization factors that are applied on the EDGE-T energy intensity and final energy values.
 #' Furthermore, the harmonization factors indicate how well the two models converged and can be used for analysis purposes.
-#'
+#' @param REMINDoutput Output of a REMIND wihtoutPlus.mif to harmonize with
 #' @param edgetOutputDir edgeTransport output directory
 #' @param baseVarSet calculated base variable set including energy intensity and final energy on fleet level
 #' @param data list of raw model output data
diff --git a/man/harmonizeOutput.Rd b/man/harmonizeOutput.Rd
index 147b1f5..d812ae0 100644
--- a/man/harmonizeOutput.Rd
+++ b/man/harmonizeOutput.Rd
@@ -4,9 +4,11 @@
 \alias{harmonizeOutput}
 \title{Harmonize data to last REMIND iteration in coupled mode}
 \usage{
-harmonizeOutput(edgetOutputDir, baseVarSet, data)
+harmonizeOutput(REMINDoutput, edgetOutputDir, baseVarSet, data)
 }
 \arguments{
+\item{REMINDoutput}{Output of a REMIND wihtoutPlus.mif to harmonize with}
+
 \item{edgetOutputDir}{edgeTransport output directory}
 
 \item{baseVarSet}{calculated base variable set including energy intensity and final energy on fleet level}

From 4f5a2e6b83650d63ac500a8cedda4fd2b7f5a125 Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Fri, 23 Aug 2024 17:11:22 +0200
Subject: [PATCH 5/7] remove duplicate variable renaming

---
 R/convertToMIF.R                              |  2 --
 inst/compareScenarios/cs_01_energy_demand.Rmd | 32 +++++++++----------
 .../cs_02_energy_services.Rmd                 |  6 ++--
 inst/compareScenarios/preprocessing.R         |  4 +--
 4 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/R/convertToMIF.R b/R/convertToMIF.R
index adfa965..30a959c 100644
--- a/R/convertToMIF.R
+++ b/R/convertToMIF.R
@@ -139,8 +139,6 @@ convertToMIF <- function(vars, GDPMER, helpers, scenario, model, gdx,  isTranspo
   toMIF[, model := model][, scenario := scenario]
   toMIF <- as.quitte(toMIF)
 
-  toMIF <- renameDuplicateVariables(toMIF)
-
   if (anyNA(toMIF)) stop("MIF output contains NAs.
                          Please check reportAndAggregatedMIF()")
   if (anyDuplicated(toMIF)) stop("MIF output contains duplicates.
diff --git a/inst/compareScenarios/cs_01_energy_demand.Rmd b/inst/compareScenarios/cs_01_energy_demand.Rmd
index 4429b14..c789eff 100644
--- a/inst/compareScenarios/cs_01_energy_demand.Rmd
+++ b/inst/compareScenarios/cs_01_energy_demand.Rmd
@@ -22,10 +22,10 @@ walk(items, showLinePlots, data = data)
 ```{r}
 tot <- "FE|Transport edge"
 items <- c(
-  "FE|Transport edge|Electricity",
-  "FE|Transport edge|Hydrogen",
-  "FE|Transport edge|Gases",
-  "FE|Transport edge|Liquids"
+  "FE|Transport|Electricity",
+  "FE|Transport|Hydrogen",
+  "FE|Transport|Gases",
+  "FE|Transport|Liquids"
   )
 showAreaAndBarPlots(data, items, tot, orderVars = "user")
 showAreaAndBarPlots(data, items, tot, fill = TRUE, orderVars = "user")
@@ -65,12 +65,12 @@ walk(items, showLinePlots, data = data)
 
 ### Passenger 
 ```{r}
-tot <- "FE|Transport edge|Pass"
+tot <- "FE|Transport|Pass"
 items <- c(
-  "FE|Transport edge|Pass|Electricity",
-  "FE|Transport edge|Pass|Hydrogen",
-  "FE|Transport edge|Pass|Gases",
-  "FE|Transport edge|Pass|Liquids"
+  "FE|Transport|Pass|Electricity",
+  "FE|Transport|Pass|Hydrogen",
+  "FE|Transport|Pass|Gases",
+  "FE|Transport|Pass|Liquids"
   )
 showAreaAndBarPlots(data, items, tot, orderVars = "user")
 showAreaAndBarPlots(data, items, tot, fill = TRUE, orderVars = "user")
@@ -190,12 +190,12 @@ walk(items, showLinePlots, data = data)
 
 ### Freight 
 ```{r}
-tot <- "FE|Transport edge|Freight"
+tot <- "FE|Transport|Freight"
 items <- c(
-  "FE|Transport edge|Freight|Electricity",
-  "FE|Transport edge|Freight|Hydrogen",
-  "FE|Transport edge|Freight|Gases",
-  "FE|Transport edge|Freight|Liquids"
+  "FE|Transport|Freight|Electricity",
+  "FE|Transport|Freight|Hydrogen",
+  "FE|Transport|Freight|Gases",
+  "FE|Transport|Freight|Liquids"
   )
 showAreaAndBarPlots(data, items, tot, orderVars = "user")
 showAreaAndBarPlots(data, items, tot, fill = TRUE, orderVars = "user")
@@ -382,7 +382,7 @@ walk(items, showLinePlots, data = data)
 ### Passenger with bunkers
 ```{r}
 tot_wbunk <- "FE|Transport|Pass with bunkers"
-tot_wobunk <- "FE|Transport edge|Pass"
+tot_wobunk <- "FE|Transport|Pass"
 items <- c(
     "FE|Transport|Bunkers|Pass|International Aviation",
     "FE|Transport|Pass|Domestic Aviation",
@@ -403,7 +403,7 @@ walk(c("FE|Transport|Pass|Aviation", items[1:5], "FE|Transport|Pass|Road|LDV", i
 ### Freight
 ```{r}
 tot_wbunk <- "FE|Transport|Freight with bunkers"
-tot_wobunk <- "FE|Transport edge|Freight"
+tot_wobunk <- "FE|Transport|Freight"
 items <- c(
       "FE|Transport|Bunkers|Freight|International Shipping",
       "FE|Transport|Freight|Domestic Shipping",
diff --git a/inst/compareScenarios/cs_02_energy_services.Rmd b/inst/compareScenarios/cs_02_energy_services.Rmd
index 24eca8f..79a9b07 100644
--- a/inst/compareScenarios/cs_02_energy_services.Rmd
+++ b/inst/compareScenarios/cs_02_energy_services.Rmd
@@ -2,7 +2,7 @@
 ## Passenger with bunkers
 ```{r}
 tot_wbunk <- "ES|Transport|Pass with bunkers"
-tot_wobunk <- "ES|Transport edge|Pass"
+tot_wobunk <- "ES|Transport|Pass"
 items <- c(
     "ES|Transport|Bunkers|Pass|International Aviation",
     "ES|Transport|Pass|Domestic Aviation",
@@ -25,7 +25,7 @@ walk(c( "ES|Transport|Pass|Aviation", tot_wobunk, items[1:5], "ES|Transport|Pass
 ## Passenger per Capita
 ```{r}
 items <- c(
-    "ES|Transport edge|Pass pCap",
+    "ES|Transport|Pass pCap",
     "ES|Transport|Pass|Aviation pCap",
     "ES|Transport|Bunkers|Pass|International Aviation pCap",
     "ES|Transport|Pass|Domestic Aviation pCap",
@@ -123,7 +123,7 @@ walk(items, showLinePlots, data=data)
 ```{r}
 
 items <- c(
-      "ES|Transport edge|Freight pCap",
+      "ES|Transport|Freight pCap",
       "ES|Transport|Bunkers|Freight|International Shipping pCap",
       "ES|Transport|Freight|Domestic Shipping pCap",
       "ES|Transport|Freight|Rail pCap",
diff --git a/inst/compareScenarios/preprocessing.R b/inst/compareScenarios/preprocessing.R
index 8cfe934..97eba14 100644
--- a/inst/compareScenarios/preprocessing.R
+++ b/inst/compareScenarios/preprocessing.R
@@ -108,7 +108,7 @@ showLinePlotsByVariable <- function(
 pCapVariables <- tribble(
   ~variable, ~newUnit, ~conversionFactor,
   "GDP|PPP", "kUS$2005", 1e6,
-  "ES|Transport edge|Pass", "pkm/yr", 1e9,
+  "ES|Transport|Pass", "pkm/yr", 1e9,
   "ES|Transport|Pass|Aviation", "pkm/yr", 1e9,
   "ES|Transport|Bunkers|Pass|International Aviation", "pkm/yr", 1e9,
   "ES|Transport|Pass|Domestic Aviation", "pkm/yr", 1e9,
@@ -117,7 +117,7 @@ pCapVariables <- tribble(
   "ES|Transport|Pass|Non-motorized|Cycle", "pkm/yr", 1e9,
   "ES|Transport|Pass|Rail|non-HSR", "pkm/yr", 1e9,
   "ES|Transport|Pass|Rail|HSR", "pkm/yr", 1e9,
-  "ES|Transport edge|Freight", "tkm/yr", 1e9,
+  "ES|Transport|Freight", "tkm/yr", 1e9,
   "ES|Transport|Bunkers|Freight|International Shipping", "tkm/yr", 1e9,
   "ES|Transport|Freight|Road", "tkm/yr", 1e9,
   "ES|Transport|Freight|Domestic Shipping", "tkm/yr", 1e9,

From e915a4cff181584f300d6f12781e24e4b3f78bfa Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Fri, 23 Aug 2024 17:40:18 +0200
Subject: [PATCH 6/7] buildlib

---
 .buildlibrary | 2 +-
 CITATION.cff  | 2 +-
 DESCRIPTION   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildlibrary b/.buildlibrary
index 44310b1..ac55dee 100644
--- a/.buildlibrary
+++ b/.buildlibrary
@@ -1,4 +1,4 @@
-ValidationKey: '199570'
+ValidationKey: '199580'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
diff --git a/CITATION.cff b/CITATION.cff
index 1d4b9b9..4be2498 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -3,7 +3,7 @@ message: If you use this software, please cite it using the metadata from this f
 type: software
 title: 'reporttransport: Reporting package for edgeTransport'
 version: 0.1.0
-date-released: '2024-08-22'
+date-released: '2024-08-23'
 abstract: This package contains edgeTransport-specific routines to report model results.
   The main functionality is to generate transport reporting variables in MIF format
   from a given edgeTransport model run folder or REMIND input data.
diff --git a/DESCRIPTION b/DESCRIPTION
index e1401be..2b582f4 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -2,7 +2,7 @@ Type: Package
 Package: reporttransport
 Title: Reporting package for edgeTransport
 Version: 0.1.0
-Date: 2024-08-22
+Date: 2024-08-23
 Authors@R:
     person("Johanna", "Hoppe", , "johanna.hoppe@pik-potsdam.de", role = c("aut", "cre"))
 Description: This package contains edgeTransport-specific routines to

From cdd3512212a05c98edcd3119caa3a8de9c1c37b1 Mon Sep 17 00:00:00 2001
From: johannah-pik <89136160+johannah-pik@users.noreply.github.com>
Date: Mon, 26 Aug 2024 12:27:12 +0200
Subject: [PATCH 7/7] bugfix in varHarmMap + tests

---
 .buildlibrary           |  2 +-
 CITATION.cff            |  2 +-
 DESCRIPTION             |  2 +-
 R/harmonizeOutput.R     |  2 +-
 R/reportEdgeTransport.R | 19 +++++++++++++++++--
 R/storeData.R           |  3 ++-
 6 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/.buildlibrary b/.buildlibrary
index ac55dee..9f3d556 100644
--- a/.buildlibrary
+++ b/.buildlibrary
@@ -1,4 +1,4 @@
-ValidationKey: '199580'
+ValidationKey: '199610'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
diff --git a/CITATION.cff b/CITATION.cff
index 4be2498..613bb64 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -3,7 +3,7 @@ message: If you use this software, please cite it using the metadata from this f
 type: software
 title: 'reporttransport: Reporting package for edgeTransport'
 version: 0.1.0
-date-released: '2024-08-23'
+date-released: '2024-08-26'
 abstract: This package contains edgeTransport-specific routines to report model results.
   The main functionality is to generate transport reporting variables in MIF format
   from a given edgeTransport model run folder or REMIND input data.
diff --git a/DESCRIPTION b/DESCRIPTION
index 2b582f4..c38c99c 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -2,7 +2,7 @@ Type: Package
 Package: reporttransport
 Title: Reporting package for edgeTransport
 Version: 0.1.0
-Date: 2024-08-23
+Date: 2024-08-26
 Authors@R:
     person("Johanna", "Hoppe", , "johanna.hoppe@pik-potsdam.de", role = c("aut", "cre"))
 Description: This package contains edgeTransport-specific routines to
diff --git a/R/harmonizeOutput.R b/R/harmonizeOutput.R
index ce4aa81..a678054 100644
--- a/R/harmonizeOutput.R
+++ b/R/harmonizeOutput.R
@@ -22,7 +22,7 @@ harmonizeOutput <- function(REMINDoutput, edgetOutputDir, baseVarSet, data) {
     harmMap <- tribble(
       ~variable,                                                  ~ all_teEs,
       "FE|Transport|Freight|Short-Medium distance|Electricity",    "te_eselt_frgt_sm",
-      "FE|Transport|Freight|Short-Medium distance|Liquids",        "te_esdie_frgt_sm",
+      "FE|Transport|Freight|Short-Medium distance|Diesel Liquids", "te_esdie_frgt_sm",
       "FE|Transport|Freight|Short-Medium distance|Hydrogen",       "te_esh2t_frgt_sm",
       "FE|Transport|Freight|Short-Medium distance|Gases",          "te_esgat_frgt_sm",
       "FE|Transport|Freight|Long distance|Diesel Liquids",         "te_esdie_frgt_lo",
diff --git a/R/reportEdgeTransport.R b/R/reportEdgeTransport.R
index 1413073..04b6436 100644
--- a/R/reportEdgeTransport.R
+++ b/R/reportEdgeTransport.R
@@ -123,13 +123,14 @@ reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NU
   #########################################################################
   # Base variable set that is needed to report REMIND input data and additional detailed transport data
   baseVarSet <- reportBaseVarSet(data = data, timeResReporting = timeResReporting)
+  # Harmonize final energy for coupled EDGE-T/REMIND model output
   if (isHarmonized) {
     REMINDoutput <- as.data.table(read.quitte(data$remindReportingFile))
     harmonizedVars <- harmonizeOutput(REMINDoutput, edgetOutputDir, baseVarSet, data)
     baseVarSet$int$fleetEnergyIntensity <- harmonizedVars$harmonizedEnergyIntensity
     baseVarSet$ext$fleetFEdemand <- harmonizedVars$harmonizedFinalEnergy
   }
-  
+
   reporting <- baseVarSet
   outputVars <- baseVarSet
 
@@ -168,7 +169,21 @@ reportEdgeTransport <- function(folderPath = file.path(".", "EDGE-T"), data = NU
                               gdx                         = data$gdxPath,
                               isTransportExtendedReported = isTransportExtendedReported)
 
-    if (isHarmonized) reporting <- reporting[!variable %in% REMINDoutput$variable]
+    if (isHarmonized) {
+      sharedVarsAfterHarmonization <- reporting[variable %in% REMINDoutput$variable]
+      if (nrow(sharedVariables) > 0) {
+        reporting <- reporting[!variable %in% REMINDoutput$variable]
+        setnames(sharedVarsAfterHarmonization, "value", "edge")
+        setnames(REMINDoutput, "value", "remind")
+        sharedVarsAfterHarmonization <- merge(sharedVarsAfterHarmonization, REMINDoutput, 
+         by = intersect(names(sharedVarsAfterHarmonization), names(REMINDoutput)), all.x = TRUE)
+        sharedVariables[, diff := (remind-edge)/remind]
+        storeData(sharedVarsAfterHarmonization)
+        message("The following variables will be dropped from the EDGE-Transport reporting because
+                they are in the REMIND reporting: ", paste(unique(sharedVarsAfterHarmonization$variable), collapse = ", "))
+        
+      }
+    }
     if (isStored) write.mif(reporting, file.path(folderPath, "Transport.mif"))
   }
 
diff --git a/R/storeData.R b/R/storeData.R
index 72c58e6..ed4cecb 100644
--- a/R/storeData.R
+++ b/R/storeData.R
@@ -47,7 +47,8 @@ storeData <- function(outputFolder, varsList = NULL, ...) {
                        "sectorESdemand",
                        "ESdemandFVsalesLevel",
                        "fleetVehiclesPerTech",
-                       "harmFactors")) subfolder <- "4_Output"
+                       "harmFactors",
+                       "sharedVarsAfterHarmonization")) subfolder <- "4_Output"
     if (varName %in% c("REMINDinputData")) subfolder <- "5_REMINDinputData"
     if (is.null(subfolder)) stop(paste0("No subfolder assigned to ", varName))