Setting the database collation via the configuration file.

- The executable comments found in the SQL files were being removed due to the
  logic in the prepareFile() method.
- Updated areas in the codebase that set the collation to set the collation via
  a value drawn from the configuration file passed to FactorBase.
- Updated mysql-extraction.txt since this appears to be the deterministic result
  when the collation is consistent for all the databases.

code/factorbase/src/main/java/ca/sfu/cs/factorbase/app/RunBB.java

@@ -52,12 +52,14 @@ public static void main(
         RuntimeLogger.logRunTime(logger, "Logger + Config Initialization", start, configEnd);
 
         long databaseStart = System.currentTimeMillis();
+        String databaseCollation = config.getProperty("dbcollation");
         FactorBaseDataBase factorBaseDatabase = new MySQLFactorBaseDataBase(
             new FactorBaseDataBaseInfo(config),
             config.getProperty("dbaddress"),
             config.getProperty("dbname"),
             config.getProperty("dbusername"),
             config.getProperty("dbpassword"),
+            databaseCollation,
             countingStrategy
         );
         RuntimeLogger.logRunTime(logger, "Creating Database Connection", databaseStart, System.currentTimeMillis());
@@ -92,6 +94,7 @@ public static void main(
         BayesBaseH.runBBH(
             factorBaseDatabase,
             globalLattice,
+            databaseCollation,
             countingStrategy
         );
         RuntimeLogger.logRunTime(logger, "Running BayesBaseH", bayesBaseHStart, System.currentTimeMillis());

code/factorbase/src/main/java/ca/sfu/cs/factorbase/database/MySQLFactorBaseDataBase.java

@@ -46,6 +46,7 @@ public class MySQLFactorBaseDataBase implements FactorBaseDataBase {
     private Connection dbConnection;
     private FactorBaseDataBaseInfo dbInfo;
     private Map<String, DataExtractor> dataExtractors;
+    private String dbCollation;
     private CountingStrategy countingStrategy;
 
 
@@ -66,10 +67,12 @@ public MySQLFactorBaseDataBase(
         String dbname,
         String username,
         String password,
+        String dbCollation,
         CountingStrategy countingStrategy
     ) throws DataBaseException {
         this.dbInfo = dbInfo;
         this.baseDatabaseName = dbname;
+        this.dbCollation = dbCollation;
         this.countingStrategy = countingStrategy;
         String baseConnectionString = MessageFormat.format(CONNECTION_STRING, dbaddress, dbname);
         Properties connectionProperties = getConnectionStringProperties(username, password);
@@ -89,20 +92,23 @@ public void setupDatabase() throws DataBaseException {
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "initialize_databases.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
 
             // Switch to start using the setup database.
             this.dbConnection.setCatalog(this.dbInfo.getSetupDatabaseName());
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metadata.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metadata_storedprocedures.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.callSP(this.dbConnection, "find_values");
@@ -111,7 +117,8 @@ public void setupDatabase() throws DataBaseException {
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
 
             // Switch to start using the BN database.
@@ -120,50 +127,59 @@ public void setupDatabase() throws DataBaseException {
             RuntimeLogger.setupLoggingTable(
                 this.dbConnection,
                 this.baseDatabaseName,
-                this.dbInfo.getBNDatabaseName()
+                this.dbInfo.getBNDatabaseName(),
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_initialize_local.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_populate.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "transfer_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "transfer_cascade.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "modelmanager_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_populate.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_RChain.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
         } catch (SQLException | IOException e) {

code/factorbase/src/main/java/ca/sfu/cs/factorbase/learning/BayesBaseH.java

@@ -108,6 +108,7 @@ public class BayesBaseH {
     public static void runBBH(
         FactorBaseDataBase database,
         RelationshipLattice globalLattice,
+        String databaseCollation,
         CountingStrategy countingStrategy
     ) throws SQLException, IOException, DataBaseException, DataExtractionException, ParsingException, ScoringException {
         initProgram(FirstRunning);
@@ -124,6 +125,7 @@ public static void runBBH(
         StructureLearning(
             database,
             con2,
+            databaseCollation,
             countingStrategy,
             globalLattice
         );
@@ -206,6 +208,7 @@ public static void runBBH(
     private static void StructureLearning(
         FactorBaseDataBase database,
         Connection conn,
+        String databaseCollation,
         CountingStrategy countingStrategy,
         RelationshipLattice lattice
     ) throws SQLException, IOException, DataBaseException, DataExtractionException, ParsingException, ScoringException {
@@ -221,7 +224,8 @@ private static void StructureLearning(
         MySQLScriptRunner.runScript(
             conn,
             Config.SCRIPTS_DIRECTORY + "modelmanager_populate.sql",
-            databaseName
+            databaseName,
+            databaseCollation
         );
 
         // Handle rnodes in a bottom-up way following the lattice.

code/factorbase/src/main/java/ca/sfu/cs/factorbase/learning/CountsManager.java

@@ -69,6 +69,7 @@ public class CountsManager {
     private static boolean generatePDPInfo;
     private static String linkCorrelation;
     private static long dbTemporaryTableSize;
+    private static String dbCollation;
     /*
      * cont is Continuous
      * ToDo: Refactor
@@ -93,7 +94,7 @@ public static void buildCT(
         RuntimeLogger.addLogEntry(dbConnection);
         try (Statement statement = dbConnection.createStatement()) {
             statement.execute("DROP SCHEMA IF EXISTS " + dbInfo.getCTDatabaseName() + ";");
-            statement.execute("CREATE SCHEMA " + dbInfo.getCTDatabaseName() + " /*M!100316 COLLATE utf8_general_ci*/;");
+            statement.execute("CREATE SCHEMA " + dbInfo.getCTDatabaseName() + " COLLATE " + dbCollation + ";");
         }
 
         // Propagate metadata based on the FunctorSet.
@@ -492,6 +493,7 @@ private static void setVarsFromConfig() {
         dbPassword = conf.getProperty("dbpassword");
         dbaddress = conf.getProperty("dbaddress");
         dbTemporaryTableSize = Math.round(1024 * 1024 * 1024 * Double.valueOf(conf.getProperty("dbtemporarytablesize")));
+        dbCollation = conf.getProperty("dbcollation");
         linkCorrelation = conf.getProperty("LinkCorrelations");
         cont = conf.getProperty("Continuous");
         String loggingLevel = conf.getProperty("LoggingLevel");

code/factorbase/src/main/java/ca/sfu/cs/factorbase/util/MySQLScriptRunner.java

@@ -32,9 +32,14 @@ private MySQLScriptRunner() {
      *
      * @param fileName - the file to create a copy of with the variables filled in.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @throws IOException if there is an issue reading from the script.
      */
-    private static String prepareFile(String fileName, String databaseName) throws IOException {
+    private static String prepareFile(
+        String fileName,
+        String databaseName,
+        String databaseCollation
+    ) throws IOException {
         InputStream inputStream = MySQLScriptRunner.class.getClassLoader().getResourceAsStream(fileName);
         if (inputStream == null) {
             throw new FileNotFoundException("Unable to read the file: " + fileName);
@@ -59,6 +64,7 @@ private static String prepareFile(String fileName, String databaseName) throws I
         String finalOutput = "";
         while (line != null) {
             line = line.replace("@database@", databaseName);
+            line = line.replace("@dbcollation@", databaseCollation);
             finalOutput += line + System.getProperty("line.separator");
             line = input.readLine();
         }
@@ -94,11 +100,17 @@ public static void callSP(Connection dbConnection, String spName) throws SQLExce
      * @param dbConnection - connection to the database to execute the script on.
      * @param scriptFileName - the path to the MySQL script to execute.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @throws SQLException if there is an issue executing the command(s).
      * @throws IOException if there is an issue reading from the script.
      */
-    public static void runScript(Connection dbConnection, String scriptFileName, String databaseName) throws SQLException, IOException {
-        runScript(dbConnection, scriptFileName, databaseName, ";");
+    public static void runScript(
+        Connection dbConnection,
+        String scriptFileName,
+        String databaseName,
+        String databaseCollation
+    ) throws SQLException, IOException {
+        runScript(dbConnection, scriptFileName, databaseName, databaseCollation, ";");
     }
 
 
@@ -110,12 +122,19 @@ public static void runScript(Connection dbConnection, String scriptFileName, Str
      * @param dbConnection - connection to the database to execute the script on.
      * @param scriptFileName - the path to the MySQL script to execute.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @param delimiter - the delimiter to use when reading the commands from the given script.
      * @throws SQLException if there is an issue executing the command(s).
      * @throws IOException if there is an issue reading from the script.
      */
-    public static void runScript(Connection dbConnection, String scriptFileName, String databaseName, String delimiter) throws SQLException, IOException {
-        String newScriptFileName = prepareFile(scriptFileName, databaseName);
+    public static void runScript(
+        Connection dbConnection,
+        String scriptFileName,
+        String databaseName,
+        String databaseCollation,
+        String delimiter
+    ) throws SQLException, IOException {
+        String newScriptFileName = prepareFile(scriptFileName, databaseName, databaseCollation);
 
         ScriptRunner runner = new ScriptRunner(dbConnection);
         try (

code/factorbase/src/main/java/ca/sfu/cs/factorbase/util/RuntimeLogger.java

@@ -94,13 +94,15 @@ public static void logExecutedQuery(Logger logger, String query) {
     public static void setupLoggingTable(
         Connection dbConnection,
         String baseDatabaseName,
-        String loggingTableDatabaseName
+        String loggingTableDatabaseName,
+        String databaseCollation
     ) throws SQLException, IOException {
         dbName = loggingTableDatabaseName;
         MySQLScriptRunner.runScript(
             dbConnection,
             Config.SCRIPTS_DIRECTORY + "logging.sql",
-            baseDatabaseName
+            baseDatabaseName,
+            databaseCollation
         );
     }
 

code/factorbase/src/main/resources/scripts/initialize_databases.sql

@@ -1,7 +1,5 @@
 -- Initialize the databases required by FactorBase.
-/*M!100316 SET collation_server = 'utf8_general_ci';*/
-
-SET collation_server = 'utf8_general_ci';
+SET collation_server = @dbcollation@;
 
 DROP SCHEMA IF EXISTS @database@_setup;
 CREATE SCHEMA @database@_setup;

config.cfg

@@ -4,6 +4,7 @@ dbname = name_of_database
 dbusername = database_username
 dbpassword = database_password
 dbtemporarytablesize = 4
+dbcollation = utf8_general_ci
 
 # FactorBase Configurations
 AutomaticSetup = 1

travis-resources/config.cfg

@@ -4,6 +4,7 @@ dbname = unielwin
 dbusername = root
 dbpassword = 123456
 dbtemporarytablesize = 4
+dbcollation = utf8_general_ci
 
 # FactorBase Configurations
 AutomaticSetup = 1

travis-resources/expected-output/mysql-extraction.txt

@@ -951,12 +951,12 @@ Table: capability(prof0,student0)_CP_pairs
 0	N/A	T	N/A
 MULT	capability(prof0,student0)	RA(prof0,student0)	salary(prof0,student0)
 Table: capability(prof0,student0)_CP_smoothed
-1	1	F	high	6	NULL	0.166667	NULL	NULL
-1	1	F	low	6	NULL	0.166667	NULL	NULL
-1	1	F	med	6	NULL	0.166667	NULL	NULL
+1	1	F	high	6	NULL	0.166665	NULL	NULL
+1	1	F	low	6	NULL	0.166665	NULL	NULL
+1	1	F	med	6	NULL	0.166665	NULL	NULL
 1	1	F	N/A	2036	NULL	0.000491	NULL	NULL
 1	1	T	high	116	NULL	0.008621	NULL	NULL
-1	1	T	N/A	6	NULL	0.166667	NULL	NULL
+1	1	T	N/A	6	NULL	0.166665	NULL	NULL
 1	2	F	high	6	NULL	0.166667	NULL	NULL
 1	2	F	low	6	NULL	0.166667	NULL	NULL
 1	2	F	med	6	NULL	0.166667	NULL	NULL
@@ -975,13 +975,13 @@ Table: capability(prof0,student0)_CP_smoothed
 1	4	T	low	56	NULL	0.017857	NULL	NULL
 1	4	T	med	96	NULL	0.010417	NULL	NULL
 1	4	T	N/A	6	NULL	0.166667	NULL	NULL
-1	5	F	high	6	NULL	0.166665	NULL	NULL
-1	5	F	low	6	NULL	0.166665	NULL	NULL
-1	5	F	med	6	NULL	0.166665	NULL	NULL
+1	5	F	high	6	NULL	0.166667	NULL	NULL
+1	5	F	low	6	NULL	0.166667	NULL	NULL
+1	5	F	med	6	NULL	0.166667	NULL	NULL
 1	5	F	N/A	2036	NULL	0.000491	NULL	NULL
 1	5	T	low	56	NULL	0.017857	NULL	NULL
 1	5	T	med	96	NULL	0.010417	NULL	NULL
-1	5	T	N/A	6	NULL	0.166665	NULL	NULL
+1	5	T	N/A	6	NULL	0.166667	NULL	NULL
 1	N/A	F	high	6	NULL	0.166667	NULL	NULL
 1	N/A	F	low	6	NULL	0.166667	NULL	NULL
 1	N/A	F	med	6	NULL	0.166667	NULL	NULL