Merge pull request #158 from keeps/bf-charset

Fixes charset problems

.gitattributes

@@ -0,0 +1 @@
+dbptk-core/src/test/resources/postgreSql/testfiles/world.sql linguist-vendored

dbptk-core/src/main/java/com/databasepreservation/Main.java

@@ -35,6 +35,7 @@ public class Main {
   public static final int EXIT_CODE_COMMAND_PARSE_ERROR = 2;
   public static final int EXIT_CODE_LICENSE_NOT_ACCEPTED = 3;
   public static final int EXIT_CODE_CONNECTION_ERROR = 4;
+  public static final int EXIT_CODE_NOT_USING_UTF8 = 5;
 
   private static final String execID = UUID.randomUUID().toString();
   public static final String APP_VERSION = getProgramVersion();
@@ -67,19 +68,26 @@ public static int internal_main(String... args) {
 
   public static int internal_main(CLI cli) {
     logProgramStart();
+    cli.logOperativeSystemInfo();
 
-    if (cli.shouldPrintHelp()) {
-      cli.printHelp();
-      return EXIT_CODE_GENERIC_ERROR;
-    }
-
-    int exitStatus = run(cli);
-    if (exitStatus == EXIT_CODE_CONNECTION_ERROR) {
-      logger.debug("Disabling encryption (for modules that support it) and trying again.");
-      cli.disableEncryption();
-      exitStatus = run(cli);
+    int exitStatus = EXIT_CODE_GENERIC_ERROR;
+    if(cli.usingUTF8()) {
+      if (cli.shouldPrintHelp()) {
+        cli.printHelp();
+      } else {
+        exitStatus = run(cli);
+        if (exitStatus == EXIT_CODE_CONNECTION_ERROR) {
+          logger.debug("Disabling encryption (for modules that support it) and trying again.");
+          cli.disableEncryption();
+          exitStatus = run(cli);
+        }
+      }
+    }else{
+      exitStatus = EXIT_CODE_NOT_USING_UTF8;
+      logger.error("The charset in use is not UTF-8.");
+      logger.error("Please try forcing UTF-8 charset by running the application with:");
+      logger.error("   java \"-Dfile.encoding=UTF-8\" -jar ...");
     }
-
     logProgramFinish(exitStatus);
 
     return exitStatus;
@@ -109,7 +117,6 @@ private static int run(CLI cli) {
 
     try {
       long startTime = System.currentTimeMillis();
-      cli.logOperativeSystemInfo();
       logger.info("Translating database: " + cli.getImportModuleName() + " to " + cli.getExportModuleName());
       importModule.getDatabase(exportModule);
       long duration = System.currentTimeMillis() - startTime;

dbptk-core/src/main/java/com/databasepreservation/cli/CLI.java

@@ -1,8 +1,11 @@
 package com.databasepreservation.cli;
 
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -63,7 +66,7 @@ public class CLI {
 
   /**
    * Create a new CLI handler
-   * 
+   *
    * @param commandLineArguments
    *          List of command line parameters as they are received by Main.main
    * @param databaseModuleFactories
@@ -86,7 +89,7 @@ public CLI(List<String> commandLineArguments, List<Class<? extends DatabaseModul
 
   /**
    * Create a new CLI handler
-   * 
+   *
    * @param commandLineArguments
    *          List of command line parameters as they are received by Main.main
    * @param databaseModuleFactories
@@ -108,8 +111,8 @@ private void includePluginModules() {
         pluginString = argsIterator.next();
         break;
       } else if (StringUtils.startsWith(arg, "--plugin=")) {
-        pluginString = arg.substring(9); // 9 is the size of the string
-                                         // "--plugin="
+        // 9 is the size of the string "--plugin="
+        pluginString = arg.substring(9);
         break;
       }
     }
@@ -133,7 +136,7 @@ private void includePluginModules() {
 
   /**
    * Gets the database import module, obtained by parsing the parameters
-   * 
+   *
    * @return The database import module specified in the parameters
    * @throws ParseException
    *           if there was an error parsing the command line parameters
@@ -167,7 +170,7 @@ public DatabaseExportModule getExportModule() throws ParseException, LicenseNotA
    * Gets the name of the export module. Note that this method does not trigger
    * the lazy loading mechanism for parsing the parameters, so the value may be
    * null if no calls to getImportModule() or getExportModule() were made.
-   * 
+   *
    * @return The export module name. null if the command line parameters have
    *         not been parsed yet
    */
@@ -285,12 +288,12 @@ private DatabaseModuleFactoriesPair getModuleFactories(List<String> args) throws
           exportModuleName = argsIterator.next();
           exportModulesFound++;
         } else if (StringUtils.startsWith(arg, "--import=")) {
-          importModuleName = arg.substring(9); // 9 is the size of the string
-                                               // "--import="
+          // 9 is the size of the string "--import="
+          importModuleName = arg.substring(9);
           importModulesFound++;
         } else if (StringUtils.startsWith(arg, "--export=")) {
-          exportModuleName = arg.substring(9); // 9 is the size of the string
-                                               // "--export="
+          // 9 is the size of the string "--export="
+          exportModuleName = arg.substring(9);
           exportModulesFound++;
         }
       }
@@ -527,7 +530,7 @@ private String printParameterHelp(String space, String shortPrefix, String longP
   /**
    * Gets the application version, as string with a prefix, ready to be included
    * in the header part of the command line help text
-   * 
+   *
    * @return the application version
    */
   public static String getApplicationVersion() {
@@ -554,6 +557,10 @@ private HashMap<String, String> getOperativeSystemInfo() {
     result.put("Java vendor", System.getProperty("java.vendor", "unknown"));
     result.put("Java version", System.getProperty("java.version", "unknown"));
     result.put("Java class version", System.getProperty("java.class.version", "unknown"));
+    // Charset.defaultCharset() is bugged on java version 5 and fixed on java 6
+    result.put("Default Charset reported by java", Charset.defaultCharset().toString());
+    result.put("Default Charset used by StreamWriter", getDefaultCharSet());
+    result.put("file.encoding property", System.getProperty("file.encoding"));
 
     return result;
   }
@@ -569,7 +576,7 @@ public void logOperativeSystemInfo() {
 
   /**
    * Prints the license text to STDOUT
-   * 
+   *
    * @param license
    *          the whole license text or some information and a link to read the
    *          full license
@@ -589,6 +596,10 @@ public boolean shouldPrintHelp() {
     }
   }
 
+  public boolean usingUTF8() {
+    return Charset.defaultCharset().equals(Charset.forName("UTF-8"));
+  }
+
   private static class DatabaseModuleFactoryNameComparator implements Comparator<DatabaseModuleFactory> {
     @Override
     public int compare(DatabaseModuleFactory o1, DatabaseModuleFactory o2) {
@@ -597,8 +608,8 @@ public int compare(DatabaseModuleFactory o1, DatabaseModuleFactory o2) {
   }
 
   private static String getUniqueOptionIdentifier(Option option) {
-    final String delimiter = "\r\f\n"; // some string that should never occur in
-                                       // option shortName nor longName
+    // some string that should never occur in option shortName nor longName
+    final String delimiter = "\r\f\n";
     return new StringBuilder().append(delimiter).append(option.getOpt()).append(delimiter).append(option.getLongOpt())
       .append(delimiter).toString();
   }
@@ -623,6 +634,12 @@ private static boolean isExportModuleOption(Option option) {
     return false;
   }
 
+  private static String getDefaultCharSet() {
+    OutputStreamWriter dummyWriter = new OutputStreamWriter(new ByteArrayOutputStream());
+    String encoding = dummyWriter.getEncoding();
+    return encoding;
+  }
+
   /**
    * Pair containing the import and export module factories
    */
@@ -633,7 +650,7 @@ public class DatabaseModuleFactoriesPair {
     /**
      * Create a new pair with an import module factory and an export module
      * factory
-     * 
+     *
      * @param importModuleFactory
      *          the import module factory
      * @param exportModuleFactory
@@ -670,7 +687,7 @@ public class DatabaseModuleFactoriesArguments {
     /**
      * Create a new pair with the import module arguments and the export module
      * arguments
-     * 
+     *
      * @param importModuleArguments
      *          import module arguments in the form Map<parameter, value parsed
      *          from the command line>