Merge branch '33-write-performance' into 'main'

Resolve "improve write performance"

Closes #33

See merge request devel/gams-transfer-r!43

CHANGELOG

@@ -1,3 +1,9 @@
+GAMS Transfer R
+==================
+- Significant performance improvements to Container write method
+- bug fix in Container read when reading a `Symbol` with unused UELs
+- bug fix in `todense` method for `Symbol`
+
 GAMS Transfer R v1.12.0
 ==================
 - released for MacOS ARM64

gamstransfer/R/Symbol.R

@@ -605,7 +605,7 @@
 
 
           idx = lapply(1:self$dimension, function(d) {
-            return(as.numeric(factor(self$records[,d], levels = levels(self$domain[[d]]$records[, 1]))) )
+            return(as.numeric(factor(self$records[,d], levels = self$domain[[d]]$records[, 1])) )
           })
 
         }

gamstransfer/src/Read.cpp

@@ -136,6 +136,7 @@ void readInternal(gdxHandle_t PGX, int varNr, bool records,
     std::fill_n(dom_uel_used[D], dom_nrecs, 0); // initialize all to false
 
     // if domain is not read before store a sym uel map
+    // sym uel map gives the place of a given uel in given domain sym id.
     if (dom_symid[D] > 0 || sym_uel_map.count(dom_symid[D]) == 0) {
       for (int k=0; k < dom_nrecs; k++) {
         if (!gdxDataReadRaw(PGX, gdx_uel_index, gdx_values, &N))
@@ -168,7 +169,7 @@ void readInternal(gdxHandle_t PGX, int varNr, bool records,
     }
   }
   else {
-
+    // indx_matrix stores positions of UELs in the domain set
     NumericMatrix indx_matrix(NrRecs, Dim);
     int n_attr;
     if (sym_type == GMS_DT_VAR || sym_type == GMS_DT_EQU) {
@@ -210,6 +211,8 @@ void readInternal(gdxHandle_t PGX, int varNr, bool records,
         }
       }
         //store domain labels
+        //dom_uel_used is true if idx positioned UEL for domain D, 
+        // is used in the symbol
         for (int D = 0; D < Dim; D++) {
           idx = GET_DOM_MAP(D, gdx_uel_index[D]);
           dom_uel_used[D][idx] = true; // set used to true
@@ -224,15 +227,10 @@ void readInternal(gdxHandle_t PGX, int varNr, bool records,
     for (int D = 0; D < Dim; D++) {
       if (dom_symid[D] < 0) continue;
 
-      // get number of uels used
+      // change dom_uel_used from true/false to position in the symbol records
       int num_used = 0;
       for (int k = 0; k < all_dom_nrecs[D]; k++) {
-        if (dom_uel_used[D][k] > 0) {
           dom_uel_used[D][k] = num_used++;
-        }
-        else {
-          dom_uel_used[D][k] = -1;
-        }
       }
 
       for (int k = 1; k <= uel_count; k++) {
@@ -253,7 +251,7 @@ void readInternal(gdxHandle_t PGX, int varNr, bool records,
 
       // create a factor v
       IntegerVector v = wrap(indx_matrix(_, D));
-
+      // Rcout << "integer vector : " << v << "\n";
       CharacterVector ch = wrap(used_uels);
       v.attr("class") = "factor";
       v.attr("levels") = ch;

gamstransfer/src/Write.cpp

@@ -27,7 +27,7 @@
 using namespace Rcpp;
 
 void WriteData(gdxHandle_t PGX, StringVector s,
-std::vector<double> V, int VarType, int Dim,
+NumericVector V, int VarType, int Dim,
 std::string elemText, std::string sym_name) {
 
   gdxStrIndexPtrs_t Indx;
@@ -97,7 +97,6 @@ std::string elemText, std::string sym_name) {
 void CPP_gdxWriteSuper(List data, LogicalVector enable, CharacterVector sysDir, 
 CharacterVector fileName, CharacterVector uel_priority, 
 bool is_uel_priority, bool compress) {
-
   std::string myUEL;
   std::string mysysDir = Rcpp::as<std::string>(sysDir);
   std::string myFileName = Rcpp::as<std::string>(fileName);
@@ -108,7 +107,6 @@ bool is_uel_priority, bool compress) {
   gdxStrIndex_t domains;
   GDXSTRINDEXPTRS_INIT(domains, domains_ptr);
 
-
   rc = gdxCreateD(&PGX, mysysDir.c_str(), Msg, sizeof(Msg));
   if (!rc) stop("CPP_gdxWriteSuper:gdxCreateD GDX init failed: %s", Msg);
 
@@ -149,18 +147,14 @@ bool is_uel_priority, bool compress) {
     if (!gdxUELRegisterDone(PGX))
       stop("CPP_gdxWriteSuper:gdxUELRegisterDone GDX error (gdxUELRegisterDone)");
   }
-
   DataFrame df;
   List domain;
   int Dim, sym_nr;
-  std::vector<double> values;
-  std::string elemText;
   StringVector colString, colElemText;
   NumericVector colDouble;
 
   sym_nr = 0;
   for (int d=0; d < data.length(); d++) {
-
     if (!enable[d]) continue;
     sym_nr++;
     Environment symname = data[d];
@@ -191,12 +185,10 @@ bool is_uel_priority, bool compress) {
     Dim = symname["dimension"];
     StringVector names(Dim);
     df = symname["records"];
-
     domain = symname["domain"];
     List domainstr;
     if (Dim != 0) {
       domainstr = symname["domainNames"];
-      std::string blah = domainstr[0];
     }
     std::string expltxt = symname["description"];
     if (!gdxDataWriteStrStart(PGX, mysym.c_str(), 
@@ -228,38 +220,45 @@ bool is_uel_priority, bool compress) {
     }
     int nrows = df.nrows();
     int ncols = df.size();
+    if (nrows == 0) {
+      if (!gdxDataWriteDone(PGX)) stop("CPP_gdxWriteSuper:gdxDataWriteDone GDX error (gdxDataWriteDone)");
+    }
+    else {
+      StringMatrix rec_domain(nrows, Dim);
 
-    for (int i =0; i < nrows; i++) {
-      for (int j=0; j < ncols; j++) {
-        if (j < Dim) {
-          colString = df[j];
-          names[j] = colString[i];
-        }
-        else {
-          if (varType != GMS_DT_SET){
-            colDouble = df[j];
-            values.push_back(colDouble[i]);
-          }
-          else {
-            colElemText = df[j];
-            values.push_back(0);
-            elemText = colElemText[i];
-          }
-        }
+      StringVector tempcol(Dim);
+      for (int d = 0; d < Dim; d++) {
+        tempcol = df[d];
+        rec_domain(_, d) = tempcol;
       }
 
+      StringVector elem_text(nrows);
+      NumericMatrix rec_vals(nrows, ncols-Dim);
 
-      if (varType != GMS_DT_SET){
-        WriteData(PGX, names, values, varType, Dim, elemText, mysym);
+      if (varType == GMS_DT_SET) {
+        elem_text = df[Dim];
       }
       else {
-        WriteData(PGX, names, values, varType, Dim, elemText, mysym);
+        NumericVector temp_num_col(Dim);
+        for (int d = Dim; d < ncols; d++) {
+          temp_num_col = df[d];
+          rec_vals(_, d-Dim) = temp_num_col;
+        }
       }
-      elemText.clear();
-      values.clear();
-    }
+      for (int i =0; i < nrows; i++) {
+        if (varType != GMS_DT_SET){
+          names = rec_domain(i, _);
+          WriteData(PGX, names, rec_vals(i, _), varType, Dim, "", mysym);
+        }
+        else {
+          names = rec_domain(i, _);
+          NumericVector zero_vec = {0};
+          WriteData(PGX, names, zero_vec, varType, Dim, Rcpp::as<std::string>(elem_text[i]), mysym);
+        }
+      }
+      if (!gdxDataWriteDone(PGX)) stop("CPP_gdxWriteSuper:gdxDataWriteDone GDX error (gdxDataWriteDone)");
 
-    if (!gdxDataWriteDone(PGX)) stop("CPP_gdxWriteSuper:gdxDataWriteDone GDX error (gdxDataWriteDone)");
+    }
 
   // get the error count
    if (gdxDataErrorCount(PGX)) {

gamstransfer/tests/testthat/test-read.R

@@ -1399,7 +1399,7 @@ a = Parameter$new(m, "a", c(hrs, mins, secs))
 # set records directly
 a$records = df
 
-# expect_true(!a$isValid())
+expect_true(a$isValid())
 }
 )
 
@@ -1453,7 +1453,7 @@ df$s_3 = factor(df$s_3, levels=levels(secs$records$uni_1), ordered = TRUE)
 # set records directly
 a$records = df
 
-# expect_true(!a$isValid())
+expect_true(a$isValid())
 }
 )
 
@@ -1849,7 +1849,7 @@ m  = Container$new()
 i = Set$new(m, "i", records=c("a", "b", "c"))
 a = Parameter$new(m, "a", i, records=data.frame(c("aa", "c"), c(1, 2)))
 
-# expect_equal(a$isValid(), FALSE)
+expect_equal(a$isValid(), TRUE)
 # expect_equal(m$listSymbols(isValid=TRUE), "i")
 # expect_equal(m$listSymbols(isValid=FALSE), "a")
 }
@@ -3323,3 +3323,18 @@ test_that("test_num_115", {
   expect_equal(m2$listSymbols(), c("h"))
 }
 )
+
+
+# test reading symbols with unused uels from gdx
+test_that("test_num_115", {
+  m = Container$new()
+  i = Set$new(m, "i", records=paste0("i", 1:5))
+  isub = Set$new(m, "isub", domain=i, records=c("i1","i2"))
+  isub$setUELs(paste0("i",1:5))
+  m$write("gt.gdx")
+
+  m2 = Container$new(testthat::test_path("gt.gdx"))
+
+  expect_equal(m["isub"]$getUELs(), paste0("i",1:5))
+}
+)