ANSI to UTF8 conversion

app/controllers/importer_controller.rb

@@ -1,5 +1,6 @@
 require 'fastercsv'
 require 'tempfile'
+require 'iconv'
 
 class MultipleIssuesForUniqueValue < Exception
 end
@@ -42,6 +43,15 @@ def match
     i = 0
     @samples = []
 
+    # transcode the ANSI format to UTF8 using iconv
+    if (iip.encoding == "A" )
+    	ansi2utf8!(iip)
+    end
+
+    # preparing the import
+    chomp_separators!(iip)
+    sub_blanks!(iip)
+
     FasterCSV.new(iip.csv_data, {:headers=>true,
     :encoding=>iip.encoding, :quote_char=>iip.quote_char, :col_sep=>iip.col_sep}).each do |row|
       @samples[i] = row
@@ -124,6 +134,8 @@ def result
       return
     end
 
+
+
     default_tracker = params[:default_tracker]
     update_issue = params[:update_issue]
     unique_field = params[:unique_field].empty? ? nil : params[:unique_field]
@@ -158,6 +170,15 @@ def result
       flash[:error] = unique_error
       return
     end
+
+    # transcode the ANSI format to UTF8 using iconv
+    if (iip.encoding == "A" )
+    	ansi2utf8!(iip)
+    end
+
+    # preparing the import
+    chomp_separators!(iip)
+    sub_blanks!(iip)
 
     FasterCSV.new(iip.csv_data, {:headers=>true, :encoding=>iip.encoding, 
         :quote_char=>iip.quote_char, :col_sep=>iip.col_sep}).each do |row|
@@ -362,6 +383,44 @@ def result
   end
 
 private
+
+	# converts the ImportInProgrsse csv data from ansi to utf8
+	# @param ImportInProgress importip, the ImportInProgress to be transcoded from ANSI to UTF8
+	def ansi2utf8!(importip)
+		converter = Iconv.new('UTF-8','CP1252')
+		importip.csv_data = converter.iconv(importip.csv_data)
+		#changing the encoding tag : U for UTF8
+		importip.encoding = "U"
+	end
+
+
+  # deletes a repetition of CSV separators at the end of the file (eg. "last_field,,,\n" becomes "last_field\n")
+  # @param ImportInProgress importip whose csv_data will be processed
+	def chomp_separators!(importip)
+		separator=importip.col_sep
+		csv_data_chomped=""
+		importip.csv_data.each_line{ |line|
+			line.chomp!
+			line=~/^ *(.*?)((#{separator})*)$/
+			csv_data_chomped+=$1+"\n"
+		}
+		importip.csv_data=csv_data_chomped
+  end
+
+	#deletes spaces around separators and  in the start of line in csv_data (prevents parsing failure)
+	# @param ImportInProgress importip whose csv_data will be cleaned from unwanted spaces
+	def sub_blanks!(importip)
+		separator=importip.col_sep
+		csv_data_subed=""
+		importip.csv_data.each_line{ |line|
+			subed_line=line.gsub(/ *#{separator} */,separator)
+			csv_data_subed += subed_line
+		}
+		importip.csv_data=csv_data_subed
+
+	end
+
+
 
   def find_project
     @project = Project.find(params[:project_id])

app/views/importer/index.html.erb

@@ -8,7 +8,7 @@
 
 	<fieldset class="box"><legend><%= l(:label_upload_format) %></legend>
 		 <p><label><%=l(:label_upload_encoding)%></label>
-		<%= select_tag "encoding", "<option value=\"U\">UTF8</option><option value=\"EUC\">EUC</option><option value=\"S\">SJIS</option><option value=\"N\">None</option>" %></p>
+		<%= select_tag "encoding", "<option value=\"A\">ANSI</option><option value=\"U\">UTF8</option><option value=\"EUC\">EUC</option><option value=\"S\">SJIS</option><option value=\"N\">None</option>" %></p>
 
 		<p><label><%=l(:label_upload_splitter)%></label>
 		<%= text_field_tag "splitter", ',', {:size => 3, :maxlength => 1}%></p>

test/ansi2utf8/ansi_accent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+t�che 1;cr�er objet;admin

test/ansi2utf8/ansi_noaccent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+tache 1;creer objet;admin

test/ansi2utf8/utf8-wBOM_accent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+tâche 1;créer objet;admin

test/ansi2utf8/utf8-wBOM_noaccent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+tache 1;creer objet;admin

test/ansi2utf8/utf8_accent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+tâche 1;créer objet;admin

test/ansi2utf8/utf8_noaccent.csv

@@ -0,0 +1,2 @@
+Subject;Description;Assigned to
+tache 1;creer objet;admin