Merge pull request #50 from qlands/master

Several fixes

AWS_build_script.txt

@@ -1,20 +1,25 @@
+#ODK Tools - Installation script tested in Ubuntu server 16.04 and 18.04
+
 sudo apt-get update
-sudo apt-get install build-essential qt5-default qtbase5-private-dev qtdeclarative5-dev cmake mongodb jq libboost-all-dev unzip zlib1g-dev automake
+sudo apt-get install build-essential qt5-default qtbase5-private-dev qtdeclarative5-dev cmake mongodb jq libboost-all-dev unzip zlib1g-dev automake npm
+
 cd /opt
 sudo git clone https://github.com/qlands/odktools.git
 sudo mkdir odktools-deps
 cd odktools-deps
 sudo wget --user=[GitHub-user] --ask-password https://github.com/mongodb/mongo-c-driver/releases/download/1.6.1/mongo-c-driver-1.6.1.tar.gz
 sudo wget --user=[GitHub-user] --ask-password https://github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz
-**Download in your local machine Quazip from https://sourceforge.net/projects/quazip/files/latest/download
-** In your local machine do: scp -i [path/to/pem.key] [path/to/]quazip-0.7.3.tar.gz [email protected]/home/ubuntu
+sudo wget --user=[GitHub-user] --ask-password https://github.com/jmcnamara/libxlsxwriter/archive/RELEASE_0.7.6.tar.gz
+sudo wget https://downloads.sourceforge.net/project/quazip/quazip/0.7.3/quazip-0.7.3.tar.gz
 sudo git clone https://github.com/rgamble/libcsv.git
+
 sudo tar xvfz mongo-c-driver-1.6.1.tar.gz
 cd mongo-c-driver-1.6.1
 sudo ./configure
 sudo make
 sudo make install
 cd ..
+
 sudo tar xvfz 1.8.4.tar.gz
 cd jsoncpp-1.8.4
 sudo mkdir build
@@ -23,39 +28,55 @@ sudo cmake ..
 sudo make
 sudo make install
 cd ../..
-sudo tar xvfz ~/quazip-0.7.3.tar.gz
+
+sudo tar xvfz quazip-0.7.3.tar.gz
 cd quazip-0.7.3
 sudo mkdir build
 cd build
 sudo cmake -DCMAKE_C_FLAGS:STRING="-fPIC" -DCMAKE_CXX_FLAGS:STRING="-fPIC" ..
 sudo make
 sudo make install
 cd ../..
+
 sudo ln -s /usr/bin/aclocal-1.15 /usr/bin/aclocal-1.14
 sudo ln -s /usr/bin/automake-1.15 /usr/bin/automake-1.14
+
+sudo tar xvfz RELEASE_0.7.6.tar.gz
+cd libxlsxwriter-RELEASE_0.7.5
+sudo mkdir build
+cd build
+sudo cmake ..
+sudo make
+sudo make install
+cd ../..
+
 cd libcsv
 sudo ./configure
 sudo make
 sudo make install
+
 cd /opt/odktools/dependencies/mongo-cxx-driver-r3.1.1
 sudo mkdir build
 cd build
 sudo cmake -DCMAKE_C_FLAGS:STRING="-O2 -fPIC" -DCMAKE_CXX_FLAGS:STRING="-O2 -fPIC" -DBSONCXX_POLY_USE_BOOST=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
 sudo make
 sudo make install
 cd ../..
+
 json2csv-cpp
 sudo qmake
 sudo make
 sudo cp json2csv /usr/bin
 cd ../../
-git submodule update --init --recursive
+
+sudo git submodule update --init --recursive
 cd 3rdparty/qjson
 sudo mkdir build
 cd build
 sudo cmake ..
 sudo make
 sudo make install
 cd ../../..
+
 sudo qmake
-sudo make 
+sudo make

JSONToMySQL/jsontomysql.pro

@@ -14,8 +14,8 @@ CONFIG   -= app_bundle
 
 TEMPLATE = app
 
-unix:INCLUDEPATH += /usr/include ../3rdparty
-unix:LIBS += -L/usr/lib64 -l qjson-qt5
+unix:INCLUDEPATH += ../3rdparty
+unix:LIBS += -l qjson-qt5
 
 SOURCES += main.cpp \
     insertvalues.cpp \

JSONToMySQL/mainclass.cpp

@@ -663,6 +663,16 @@ QList<TfieldDef > mainClass::createSQL(QSqlDatabase db, QVariantMap jsonData, QS
             key.multiSelectTable = fields[pos].multiSelectTable;
             key.value = jsonData[fields[pos].xmlCode].toString();
             key.value = key.value.simplified();
+            if ((fields[pos].ODKType == "start") || (fields[pos].ODKType == "end") || (fields[pos].ODKType == "time") || (fields[pos].ODKType == "datetime"))
+            {
+                if (key.value.indexOf(".") >= 0)
+                {
+                    QStringList parts;
+                    parts = key.value.split(".");
+                    key.value = parts[0];
+                    key.value = key.value.replace("T"," ");
+                }
+            }
             key.uuid = table + "~" + strRecordUUID;
 
             // If its empty. The try to find it in jsonData2
@@ -756,16 +766,16 @@ QList<TfieldDef > mainClass::createSQL(QSqlDatabase db, QVariantMap jsonData, QS
 
                 //sqlValues = sqlValues + "'" + fixString(fieldValue) + "',";
             }            
-            if ((fields[pos].ODKType == "start") || (fields[pos].ODKType == "end"))
+            if ((fields[pos].ODKType == "start") || (fields[pos].ODKType == "end") || (fields[pos].ODKType == "time") || (fields[pos].ODKType == "datetime"))
             {
                 if (fieldValue.indexOf(".") >= 0)
                 {
                     QStringList parts;
                     parts = fieldValue.split(".");
                     fieldValue = parts[0];
-                    fieldValue.replace("T"," ");
+                    fieldValue = fieldValue.replace("T"," ");
                 }
-            }
+            }            
             insValue.value = fieldValue;
             insertObject.insertValue(insValue);
         }

ODKToMySQL/main.cpp

@@ -1100,7 +1100,7 @@ void genSQL(QString ddlFile,QString insFile, QString metaFile, QString xmlFile,
         else
             sql = sql + "\n";
 
-        sqlCreateStrm << sql; //Add the final SQL to the DDL file
+        sqlCreateStrm << sql; //Add the triggers to the SQL DDL file
 
         //Create the inserts of the lookup tables values into the insert SQL
         if (tables[pos].lkpValues.count() > 0)
@@ -1132,8 +1132,8 @@ void genSQL(QString ddlFile,QString insFile, QString metaFile, QString xmlFile,
                 }
             }
         }
-
     }
+
     sqlInsertStrm << "COMMIT;\n";
     //Create the manifext file. If exist it get overwriten
     if (QFile::exists(xmlFile))
@@ -1244,6 +1244,10 @@ TfieldMap mapODKFieldTypeToMySQL(QString ODKFieldType)
     {
         result.type = "time";
     }
+    if (ODKFieldType == "datetime")
+    {
+        result.type = "datetime";
+    }
     if (ODKFieldType == "geopoint")
     {
         result.type = "varchar";
@@ -1621,6 +1625,13 @@ QList<TlkpValue> getSelectValuesFromCSV(QString searchExpresion, QXlsx::Workshee
                                 desc.desc = query.value(fixColumnName(descColumns[pos].desc)).toString();
                                 value.desc.append(desc);
                             }
+                            else
+                            {
+                                TlngLkpDesc desc;
+                                desc.langCode = descColumns[pos].langCode;
+                                desc.desc = "";
+                                value.desc.append(desc);
+                            }
                         }
                         res.append(value);
                     }
@@ -1860,6 +1871,7 @@ int processXLSX(QString inputFile, QString mainTable, QString mainField, QDir di
         QXlsx::CellReference langRef;
         QXlsx::Cell *cell;
         QXlsx::Worksheet *excelSheet = (QXlsx::Worksheet*)xlsx.sheet("settings");
+        bool hasDefaultLanguage = false;
         for (ncols = 1; ncols <= excelSheet->dimension().lastColumn(); ncols++)
         {
             ref.setRow(1);
@@ -1872,6 +1884,7 @@ int processXLSX(QString inputFile, QString mainTable, QString mainField, QDir di
                     ref.setRow(2);
                     ref.setColumn(ncols);
                     ODKLanguages.append(excelSheet->cellAt(ref)->value().toString());
+                    hasDefaultLanguage = true;
                 }
         }
 
@@ -1895,6 +1908,14 @@ int processXLSX(QString inputFile, QString mainTable, QString mainField, QDir di
                         if (ODKLanguages.indexOf(language) < 0)
                             ODKLanguages.append(language);
                     }
+                    else
+                    {
+                        if (!hasDefaultLanguage)
+                        {
+                            if (ODKLanguages.indexOf("English") < 0)
+                                ODKLanguages.append("English");
+                        }
+                    }
                 }
             }
         }
@@ -2160,7 +2181,7 @@ int processXLSX(QString inputFile, QString mainTable, QString mainField, QDir di
         {
             TlngLkpDesc langDesc;
             langDesc.langCode = languages[lang].code;
-            langDesc.desc = "Survey ID (UUID)";
+            langDesc.desc = "Submission ID";
             fsurveyID.desc.append(langDesc);
         }
         fsurveyID.type = "varchar";
@@ -2183,7 +2204,7 @@ int processXLSX(QString inputFile, QString mainTable, QString mainField, QDir di
         {
             TlngLkpDesc langDesc;
             langDesc.langCode = languages[lang].code;
-            langDesc.desc = "Origin ID: formhub or aggregate";
+            langDesc.desc = "Origin";
             foriginID.desc.append(langDesc);
         }
         foriginID.type = "varchar";

ODKToMySQL/odktomysql.pro

@@ -14,8 +14,11 @@ CONFIG   -= app_bundle
 
 TEMPLATE = app
 
-unix:INCLUDEPATH += ../3rdparty /usr/local/include /usr/include
-unix:LIBS += -L/lib64 -L/usr/lib64 -L/usr/local/lib -lquazip5 -lcsv
+#unix:INCLUDEPATH += ../3rdparty /usr/local/include /usr/include
+#unix:LIBS += -L/lib64 -L/usr/lib64 -L/usr/local/lib -lquazip5 -lcsv
+
+unix:INCLUDEPATH += ../3rdparty
+unix:LIBS += -lquazip5 -lcsv
 
 
 include(../3rdparty/QtXlsxWriter/src/xlsx/qtxlsx.pri)

README.md

@@ -286,43 +286,9 @@ JSON to CSV creates a CSV file based a JSON file by flattening the JSON structur
 
 
 ## Building and testing
-To build ODKTools on Ubuntu Server 16.04.3 LTS do:
-
-    $ sudo apt-get update
-    $ sudo apt-get install build-essential qt5-default qtbase5-private-dev qtdeclarative5-dev cmake mongodb jq libboost-all-dev unzip zlib1g-dev automake
-
-You also need to build and install:
-
- - [Mongo C Driver](https://github.com/mongodb/mongo-c-driver/releases/download/1.6.1/mongo-c-driver-1.6.1.tar.gz)
- - [JSONCpp](https://github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz)
- - [QuaZip](http://quazip.sourceforge.net/) using -DCMAKE_C_FLAGS:STRING="-fPIC" -DCMAKE_CXX_FLAGS:STRING="-fPIC"
- - [LibCSV](https://github.com/rgamble/libcsv) . It uses automake 1.14 so create a symbolic link of aclocal and automake from your version to 1.14
-
-To build ODK Tools do:
-
-        $ git clone https://github.com/ilri/odktools.git
-        $ cd odktools
-        $ git submodule update --init --recursive
-        $ cd dependencies/mongo-cxx-driver-r3.1.1/build
-        $ cmake -DCMAKE_C_FLAGS:STRING="-O2 -fPIC" -DCMAKE_CXX_FLAGS:STRING="-O2 -fPIC" -DBSONCXX_POLY_USE_BOOST=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
-        $ sudo make install
-        $ cd ../../..
-        $ cd dependencies/json2csv-cpp
-        $ qmake
-        $ make
-        $ sudo cp json2csv /usr/bin
-        $ cd ../..
-        $ cd 3rdparty/qjson
-        $ mkdir build
-        $ cd build
-        $ cmake ..
-        $ make
-        $ sudo make install
-        $ cd ../../..
-        $ qmake
-        $ make
-
+Installation instructions for Ubuntu Server 16.04 or 18.04 are available [here](https://github.com/qlands/odktools/blob/master/AWS_build_script.txt)
 
+
 ## Author
 Carlos Quiros (cquiros_at_qlands.com / c.f.quiros_at_cgiar.org)
 
@@ -335,4 +301,4 @@ This repository contains the code of:
 - [MongoDB C++ Driver](https://mongodb.github.io/mongo-cxx-driver/) which is licensed under the [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
 - [JSON2CSV-CPP](https://github.com/once-ler/json2csv-cpp) which does not have a license. We assume MIT or BSD.
 
-Otherwise, ODKTools is licensed under [LGPL V3](http://www.gnu.org/licenses/lgpl-3.0.html).
+Otherwise, ODKTools is licensed under [LGPL V3](http://www.gnu.org/licenses/lgpl-3.0.html).

XMLtoJSON/xmltojson.pro

@@ -18,6 +18,7 @@ SOURCES += main.cpp
 # depend on your compiler). Please consult the documentation of the
 # deprecated API in order to know how to port your code away from it.
 DEFINES += QT_DEPRECATED_WARNINGS
+QMAKE_CFLAGS += $$(CFLAGS) QMAKE_CFLAGS_ISYSTEM=-I
 
 # You can also make your code fail to compile if you use deprecated APIs.
 # In order to do so, uncomment the following line.