Skip to content

Commit

Permalink
bugfix and revert mapping
Browse files Browse the repository at this point in the history
  • Loading branch information
caviddhen committed Apr 19, 2024
1 parent d67458b commit b4b707f
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .buildlibrary
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ValidationKey: '27862555'
ValidationKey: '27883792'
AcceptedWarnings:
- 'Warning: package ''.*'' was built under R version'
- 'Warning: namespace ''.*'' is not available and has been replaced'
Expand Down
4 changes: 2 additions & 2 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ cff-version: 1.2.0
message: If you use this software, please cite it using the metadata from this file.
type: software
title: 'mrcommons: MadRat commons Input Data Library'
version: 1.40.5
date-released: '2024-04-18'
version: 1.40.6
date-released: '2024-04-19'
abstract: Provides useful functions and a common structure to all the input data required
to run models like MAgPIE and REMIND of model input data.
authors:
Expand Down
4 changes: 2 additions & 2 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
Package: mrcommons
Type: Package
Title: MadRat commons Input Data Library
Version: 1.40.5
Date: 2024-04-18
Version: 1.40.6
Date: 2024-04-19
Authors@R: c(person("Benjamin Leon", "Bodirsky", email = "[email protected]", role = "aut"),
person("Kristine", "Karstens", role = "aut"),
person("Lavinia", "Baumstark", role = "aut"),
Expand Down
66 changes: 33 additions & 33 deletions R/readFAOTradeMatrix.R
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

readFAOTradeMatrix <- function(subtype) { # nolint

file <- "Trade_DetailedTradeMatrix_E_All_Data_(Normalized).csv"
file <- "Trade_DetailedTradeMatrix_E_All_Data_(Normalized).csv"

# ---- Select columns to be read from file and read file ----

Expand All @@ -39,17 +39,17 @@ file <- "Trade_DetailedTradeMatrix_E_All_Data_(Normalized).csv"
factorCols <- c("Reporter.Country.Code", "Partner.Country.Code", "Item.Code", "Element.Code", "Element")
readcolClass[csvcolnames %in% factorCols] <- "factor"
readcolClass[csvcolnames %in% c("Area", "Country", "Element", "Item", "Unit",
"Months", "Reporter.Countries", "Partner.Countries")] <- "character"
"Months", "Reporter.Countries", "Partner.Countries")] <- "character"
readcolClass[csvcolnames %in% c("Value", "Year")] <- NA
if (!long) readcolClass[grepl("Y[0-9]{4}$", csvcolnames)] <- NA

fao <- suppressWarnings(
fread(input = file, header = FALSE, skip = 1, sep = ",",
colClasses = readcolClass,
col.names = csvcolnames[is.na(readcolClass) | readcolClass != "NULL"],
quote = "\"",
encoding = "Latin-1", showProgress = FALSE,
))
fread(input = file, header = FALSE, skip = 1, sep = ",",
colClasses = readcolClass,
col.names = csvcolnames[is.na(readcolClass) | readcolClass != "NULL"],
quote = "\"",
encoding = "Latin-1", showProgress = FALSE
))
fao <- as.data.frame(fao)
# from wide to long (move years from individual columns into one column)
if (!long) {
Expand Down Expand Up @@ -103,63 +103,63 @@ file <- "Trade_DetailedTradeMatrix_E_All_Data_(Normalized).csv"

# some small islands correspond to the same ISO3code, just remove them for now
fao <- filter(fao, !.data$ReporterCountries %in% c("Johnston Island", "Midway Island",
"Canton and Enderbury Islands", "Wake Island"),
!.data$PartnerCountries %in% c("Johnston Island", "Midway Island",
"Canton and Enderbury Islands", "Wake Island"))
"Canton and Enderbury Islands", "Wake Island"),
!.data$PartnerCountries %in% c("Johnston Island", "Midway Island",
"Canton and Enderbury Islands", "Wake Island"))

fao <- unite(fao, col = "ISO", c(.data$ReporterISO, .data$PartnerISO), sep = ".", remove = FALSE)
fao <- unite(fao, col = "ISO", c(.data$ReporterISO, .data$PartnerISO), sep = ".", remove = FALSE)

# subset by both trade column and product column
mapping <- toolGetMapping("newFAOitems_online_DRAFT.csv", type = "sectoral", where = "mrcommons")
mapping <- mapping[, c("new_FAOoriginalItem_fromWebsite", "k")]
colnames(mapping)[1] <- "ItemCodeItem"
mapping <- distinct(mapping)
# subset by both trade column and product column
mapping <- toolGetMapping("newFAOitems_online_DRAFT.csv", type = "sectoral", where = "mrcommons")
mapping <- mapping[, c("new_FAOoriginalItem_fromWebsite", "k")]
colnames(mapping)[1] <- "ItemCodeItem"
mapping <- distinct(mapping)


fao <- inner_join(fao, mapping)
fao <- inner_join(fao, mapping)

kcr <- findset("kcr")
kli <- findset("kli")
kothers <- setdiff(findset("kall"), c(kcr, kli))
kcr <- findset("kcr")
kli <- findset("kli")
kothers <- setdiff(findset("kall"), c(kcr, kli))

elements <- list(
import_value_kcr = list(trade = "import_kUS$", product = kcr),
import_value_kli = list(trade = "import_kUS$", product = kli),
import_value_kothers = list(trade = "import_kUS$", product = kothers),
import_qty_kcr = list(trade = c("import", "Import_Quantity_(1000_Head)",
"Import_Quantity_(Head)", "Import_Quantity_(no)"),
product = kcr),
product = kcr),
import_qty_kli = list(trade = c("import", "Import_Quantity_(1000_Head)",
"Import_Quantity_(Head)", "Import_Quantity_(no)"),
product = kli),
import_qty_kothers = list(trade = c("import", "Import_Quantity_(1000_Head)",
"Import_Quantity_(Head)", "Import_Quantity_(no)"),
product = kothers),
"Import_Quantity_(Head)", "Import_Quantity_(no)"),
product = kothers),
export_value_kcr = list(trade = "export_kUS$", product = kcr),
export_value_kli = list(trade = "export_kUS$", product = kli),
export_value_kothers = list(trade = "export_kUS$", product = kothers),
export_qty_kcr = list(trade = c("export", "Export_Quantity_(1000_Head)",
"Export_Quantity_(Head)", "Export_Quantity_(no)"),
product = kcr),
product = kcr),
export_qty_kli = list(trade = c("export", "Export_Quantity_(1000_Head)",
"Export_Quantity_(Head)", "Export_Quantity_(no)"),
product = kli),
export_qty_kothers = list(trade = c("export", "Export_Quantity_(1000_Head)",
"Export_Quantity_(Head)", "Export_Quantity_(no)"),
product = kothers)
)
product = kothers)
)

element <- toolSubtypeSelect(subtype, elements)
element <- toolSubtypeSelect(subtype, elements)

out <- filter(fao, .data$ElementShort %in% element$trade, .data$k %in% element$product)
out <- filter(fao, .data$ElementShort %in% element$trade, .data$k %in% element$product)

out <- as.magpie(out[, c("Year", "ISO", "ItemCodeItem", "ElementShort", "Value")],
out <- as.magpie(out[, c("Year", "ISO", "ItemCodeItem", "ElementShort", "Value")],
temporal = 1, spatial = 2, datacol = 5) # import/export unit is in tonnes
getItems(out, dim = 1, raw = TRUE) <- gsub("_", ".", getItems(out, dim = 1))
gc()
getItems(out, dim = 1, raw = TRUE) <- gsub("_", ".", getItems(out, dim = 1))
gc()


out <- magpiesort(out)
out <- magpiesort(out)

return(out)
}
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# MadRat commons Input Data Library

R package **mrcommons**, version **1.40.5**
R package **mrcommons**, version **1.40.6**

[![CRAN status](https://www.r-pkg.org/badges/version/mrcommons)](https://cran.r-project.org/package=mrcommons) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3822009.svg)](https://doi.org/10.5281/zenodo.3822009) [![R build status](https://github.com/pik-piam/mrcommons/workflows/check/badge.svg)](https://github.com/pik-piam/mrcommons/actions) [![codecov](https://codecov.io/gh/pik-piam/mrcommons/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrcommons) [![r-universe](https://pik-piam.r-universe.dev/badges/mrcommons)](https://pik-piam.r-universe.dev/builds)

Expand Down Expand Up @@ -39,7 +39,7 @@ In case of questions / problems please contact Jan Philipp Dietrich <dietrich@pi

To cite package **mrcommons** in publications use:

Bodirsky B, Karstens K, Baumstark L, Weindl I, Wang X, Mishra A, Wirth S, Stevanovic M, Steinmetz N, Kreidenweis U, Rodrigues R, Popov R, Humpenoeder F, Giannousakis A, Levesque A, Klein D, Araujo E, Beier F, Oeser J, Pehl M, Leip D, Crawford M, Molina Bacca E, von Jeetze P, Martinelli E, Schreyer F, Soergel B, Sauer P, Hötten D, Hasse R, Abrahão G, Weigmann P, Dietrich J (2024). _mrcommons: MadRat commons Input Data Library_. doi:10.5281/zenodo.3822009 <https://doi.org/10.5281/zenodo.3822009>, R package version 1.40.5, <https://github.com/pik-piam/mrcommons>.
Bodirsky B, Karstens K, Baumstark L, Weindl I, Wang X, Mishra A, Wirth S, Stevanovic M, Steinmetz N, Kreidenweis U, Rodrigues R, Popov R, Humpenoeder F, Giannousakis A, Levesque A, Klein D, Araujo E, Beier F, Oeser J, Pehl M, Leip D, Crawford M, Molina Bacca E, von Jeetze P, Martinelli E, Schreyer F, Soergel B, Sauer P, Hötten D, Hasse R, Abrahão G, Weigmann P, Dietrich J (2024). _mrcommons: MadRat commons Input Data Library_. doi: 10.5281/zenodo.3822009 (URL: https://doi.org/10.5281/zenodo.3822009), R package version 1.40.6, <URL: https://github.com/pik-piam/mrcommons>.

A BibTeX entry for LaTeX users is

Expand All @@ -48,7 +48,7 @@ A BibTeX entry for LaTeX users is
title = {mrcommons: MadRat commons Input Data Library},
author = {Benjamin Leon Bodirsky and Kristine Karstens and Lavinia Baumstark and Isabelle Weindl and Xiaoxi Wang and Abhijeet Mishra and Stephen Wirth and Mishko Stevanovic and Nele Steinmetz and Ulrich Kreidenweis and Renato Rodrigues and Roman Popov and Florian Humpenoeder and Anastasis Giannousakis and Antoine Levesque and David Klein and Ewerton Araujo and Felicitas Beier and Julian Oeser and Michaja Pehl and Debbora Leip and Michael Crawford and Edna {Molina Bacca} and Patrick {von Jeetze} and Eleonora Martinelli and Felix Schreyer and Bjoern Soergel and Pascal Sauer and David Hötten and Robin Hasse and Gabriel Abrahão and Pascal Weigmann and Jan Philipp Dietrich},
year = {2024},
note = {R package version 1.40.5},
note = {R package version 1.40.6},
doi = {10.5281/zenodo.3822009},
url = {https://github.com/pik-piam/mrcommons},
}
Expand Down
8 changes: 4 additions & 4 deletions inst/extdata/FAOelementShort.csv
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ ElementCode;Element;ElementShort;Unit;Unit_original
5521;Feed;feed;tonnes;1000 tonnes
5525;Seed;seed;tonnes;tonnes
5527;Seed;seed;tonnes;1000 tonnes
5610;Import Quantity;import;t;tonnes
5611;Import Quantity;import;t;1000 tonnes
5610;Import Quantity;import;tonnes;tonnes
5611;Import Quantity;import;tonnes;1000 tonnes
5616;Import Quantity;import_m3;m3;m3
5622;Import Value;import_kUS$;1000 US$;1000 US$
5910;Export Quantity;export;t;tonnes
5911;Export Quantity;export;t;1000 tonnes
5910;Export Quantity;export;tonnes;tonnes
5911;Export Quantity;export;tonnes;1000 tonnes
5916;Export Quantity;export_m3;m3;m3
5922;Export Value;export_kUS$;1000 US$;1000 US$
15 changes: 0 additions & 15 deletions inst/extdata/sectoral/newFAOitems_online_DRAFT.csv
Original file line number Diff line number Diff line change
Expand Up @@ -650,12 +650,10 @@
638,,1293|Crude organic material n.e.c.,,,,,,,,
639,,"831|Other manufactured tobacco and manufactured tobacco substitutes; ""homogenized"" or ""reconstituted"" tobacco; tobacco extracts and essences",,,,,,,,
640,,1232|Food preparations n.e.c.,,,,,,,,
641,,238|Cake of soya beans,,,,,,,,
642,,"1169|Other live animals non food, n.e.c.",,,,,,,,
643,,677|Hop cones,,,,,,,,
644,,"767|Cotton lint, ginned",,,,,,,,
645,,826|Unmanufactured tobacco,,,,,,,,
646,,332|Cake of cottonseed,,,,,,,,
647,,652|Vegetable products for feed n.e.c.,,,,,,,,
648,,1057|Chickens,,,,,,,,
649,,"30|Rice, paddy (rice milled equivalent)",,,,,,rice_pro,rice_pro,new bilateral trade matrix product
Expand Down Expand Up @@ -684,32 +682,26 @@
672,,769|Cotton waste,,,,,,,,
673,,"651|Other forage products, n.e.c.",,,,,,,,
674,,"628|Pulp, waste of fruit for feed",,,,,,,,
675,,245|Cake of groundnuts,,,,,,,,
676,,"987|Shorn wool, greasy, including fleece-washed shorn wool",,,,,,,,
677,,"754|Pyrethrum, dried flowers",,,,,,,,
678,,"780|Jute, raw or retted",,,,,,,,
679,,845|Compound feed n.e.c.,,,,,,,,
680,,862|Lucerne (alfalfa) meal and pellets,,,,,,,,
681,,1096|Horses,,,,,,,,
682,,335|Cake of linseed,,,,,,,,
683,,"1216|Other raw skins of other animals, preserved",,,,,,,,
684,,169|Beet pulp,,,,,,,,
685,,"1009|Wool, hair waste",,,,,,,,
686,,1034|Swine / pigs,,,,,,,,
687,,"928|Skins, wet-salted of calves",,,,,,,,
688,,253|Cake of copra,,,,,,,,
689,,"821|Other fibre crops, raw, n.e.c.",,,,,,,,
690,,269|Cake of sunflower seed,,,,,,,,
691,,946|Buffalo,,,,,,,,
692,,"809|Abaca, manila hemp, raw",,,,,,,,
693,,1126|Camels,,,,,,,,
694,,"748|Peppermint, spearmint",,,,,,,,
695,,259|Cake of palm kernel,,,,,,,,
696,,"813|Coir, raw",,,,,,,,
697,,"1026|Skins, wet-salted of goats",,,,,,,,
698,,"773|Flax, processed but not spun",,,,,,,,
699,,"771|Flax, raw or retted",,,,,,,,
700,,272|Cake of rapeseed,,,,,,,,
701,,1173|Meat meal,,,,,,,,
702,,1068|Ducks,,,,,,,,
703,,770|Cotton linters,,,,,,,,
Expand All @@ -724,20 +716,16 @@
712,,1140|Rabbits and hares,,,,,,,,
713,,755|Pyrethrum extract,,,,,,,,
714,,"774|Flax, tow and waste",,,,,,,,
715,,61|Cake of maize,,,,,,,,
716,,1110|Mules and hinnies,,,,,,,,
717,,1079|Turkeys,,,,,,,,
718,,1083|Other birds,,,,,,,,
719,,"996|Skins, wet-salted of sheep",,,,,,,,
720,,1185|Silk-worm cocoons suitable for reeling,,,,,,,,
721,,294|Cake of mustard seed,,,,,,,,
722,,1031|Coarse goat hair,,,,,,,,
723,,37|Cake of rice bran,,,,,,,,
724,,"1104|Hides, dry-salted of horses",,,,,,,,
725,,1157|Other camelids,,,,,,,,
726,,"859|Hay for forage, from other crops n.e.c.",,,,,,,,
727,,855|Feed minerals,,,,,,,,
728,,291|Cake of sesame seed,,,,,,,,
729,,120|Potato offals,,,,,,,,
730,,"858|Hay for forage, from legumes",,,,,,,,
731,,"958|Hides, wet-salted of buffalo",,,,,,,,
Expand All @@ -746,9 +734,6 @@
734,,"1213|Other raw skins of other animals, fresh",,,,,,,,
735,,"778|Kapok fibre, raw",,,,,,,,
736,,853|Vitamins,,,,,,,,
737,,338|Cake of hempseed,,,,,,,,
738,,"646|Turnips, for forage",,,,,,,,
739,,"1134|Hides, wet-salted of camels",,,,,,,,
740,,282|Cake of safflowerseed,,,,,,,,
741,,314|Cake of kapok,,,,,,,,
742,,630|Cane tops,,,,,,,,

0 comments on commit b4b707f

Please sign in to comment.