diff --git a/app/controllers/importer_controller.rb b/app/controllers/importer_controller.rb index 38b32a1..ec39f0c 100644 --- a/app/controllers/importer_controller.rb +++ b/app/controllers/importer_controller.rb @@ -1,5 +1,6 @@ require 'fastercsv' require 'tempfile' +require 'iconv' class MultipleIssuesForUniqueValue < Exception end @@ -42,6 +43,15 @@ def match i = 0 @samples = [] + # transcode the ANSI format to UTF8 using iconv + if (iip.encoding == "A" ) + ansi2utf8!(iip) + end + + # preparing the import + chomp_separators!(iip) + sub_blanks!(iip) + FasterCSV.new(iip.csv_data, {:headers=>true, :encoding=>iip.encoding, :quote_char=>iip.quote_char, :col_sep=>iip.col_sep}).each do |row| @samples[i] = row @@ -124,6 +134,8 @@ def result return end + + default_tracker = params[:default_tracker] update_issue = params[:update_issue] unique_field = params[:unique_field].empty? ? nil : params[:unique_field] @@ -158,6 +170,15 @@ def result flash[:error] = unique_error return end + + # transcode the ANSI format to UTF8 using iconv + if (iip.encoding == "A" ) + ansi2utf8!(iip) + end + + # preparing the import + chomp_separators!(iip) + sub_blanks!(iip) FasterCSV.new(iip.csv_data, {:headers=>true, :encoding=>iip.encoding, :quote_char=>iip.quote_char, :col_sep=>iip.col_sep}).each do |row| @@ -362,6 +383,44 @@ def result end private + + # converts the ImportInProgrsse csv data from ansi to utf8 + # @param ImportInProgress importip, the ImportInProgress to be transcoded from ANSI to UTF8 + def ansi2utf8!(importip) + converter = Iconv.new('UTF-8','CP1252') + importip.csv_data = converter.iconv(importip.csv_data) + #changing the encoding tag : U for UTF8 + importip.encoding = "U" + end + + + # deletes a repetition of CSV separators at the end of the file (eg. "last_field,,,\n" becomes "last_field\n") + # @param ImportInProgress importip whose csv_data will be processed + def chomp_separators!(importip) + separator=importip.col_sep + csv_data_chomped="" + importip.csv_data.each_line{ |line| + line.chomp! + line=~/^ *(.*?)((#{separator})*)$/ + csv_data_chomped+=$1+"\n" + } + importip.csv_data=csv_data_chomped + end + + #deletes spaces around separators and in the start of line in csv_data (prevents parsing failure) + # @param ImportInProgress importip whose csv_data will be cleaned from unwanted spaces + def sub_blanks!(importip) + separator=importip.col_sep + csv_data_subed="" + importip.csv_data.each_line{ |line| + subed_line=line.gsub(/ *#{separator} */,separator) + csv_data_subed += subed_line + } + importip.csv_data=csv_data_subed + + end + + def find_project @project = Project.find(params[:project_id]) diff --git a/app/views/importer/index.html.erb b/app/views/importer/index.html.erb index 9261c6f..048face 100644 --- a/app/views/importer/index.html.erb +++ b/app/views/importer/index.html.erb @@ -8,7 +8,7 @@
<%= l(:label_upload_format) %>

- <%= select_tag "encoding", "" %>

+ <%= select_tag "encoding", "" %>

<%= text_field_tag "splitter", ',', {:size => 3, :maxlength => 1}%>

diff --git a/test/ansi2utf8/ansi_accent.csv b/test/ansi2utf8/ansi_accent.csv new file mode 100644 index 0000000..5c3641c --- /dev/null +++ b/test/ansi2utf8/ansi_accent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tâche 1;créer objet;admin \ No newline at end of file diff --git a/test/ansi2utf8/ansi_noaccent.csv b/test/ansi2utf8/ansi_noaccent.csv new file mode 100644 index 0000000..a40cb32 --- /dev/null +++ b/test/ansi2utf8/ansi_noaccent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tache 1;creer objet;admin \ No newline at end of file diff --git a/test/ansi2utf8/utf8-wBOM_accent.csv b/test/ansi2utf8/utf8-wBOM_accent.csv new file mode 100644 index 0000000..e8cdf1b --- /dev/null +++ b/test/ansi2utf8/utf8-wBOM_accent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tâche 1;créer objet;admin \ No newline at end of file diff --git a/test/ansi2utf8/utf8-wBOM_noaccent.csv b/test/ansi2utf8/utf8-wBOM_noaccent.csv new file mode 100644 index 0000000..47b59d7 --- /dev/null +++ b/test/ansi2utf8/utf8-wBOM_noaccent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tache 1;creer objet;admin \ No newline at end of file diff --git a/test/ansi2utf8/utf8_accent.csv b/test/ansi2utf8/utf8_accent.csv new file mode 100644 index 0000000..2088f68 --- /dev/null +++ b/test/ansi2utf8/utf8_accent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tâche 1;créer objet;admin \ No newline at end of file diff --git a/test/ansi2utf8/utf8_noaccent.csv b/test/ansi2utf8/utf8_noaccent.csv new file mode 100644 index 0000000..a40cb32 --- /dev/null +++ b/test/ansi2utf8/utf8_noaccent.csv @@ -0,0 +1,2 @@ +Subject;Description;Assigned to +tache 1;creer objet;admin \ No newline at end of file