Merge pull request #11 from lyrasis/dev

Improvements to CSV converters and Clean::ClearFields
lyrasis · Mar 19, 2021 · 7c40d6b · 7c40d6b
2 parents 5b3a0a2 + 56651fa
commit 7c40d6b
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 6 deletions.
diff --git a/lib/kiba/extend.rb b/lib/kiba/extend.rb
@@ -37,10 +37,14 @@ module Extend
       begin
         if s.nil?
           nil
+        elsif s == 'NULL'
+          nil
         else
           s.strip
             .gsub(/  +/, ' ')
             .sub(/,$/, '')
+            .sub(/^%(LINEBREAK|CRLF)%/, '')
+            .sub(/%(LINEBREAK|CRLF)%$/, '')
             .strip
         end
       rescue ArgumentError

diff --git a/lib/kiba/extend/transforms/clean.rb b/lib/kiba/extend/transforms/clean.rb
@@ -34,12 +34,19 @@ def process(row)
         end
 
         class ClearFields
-          def initialize(fields:)
+          def initialize(fields:, if_equals: nil)
             @fields = fields
+            @if_equals = if_equals
           end
 
           def process(row)
-            @fields.each{ |field| row[field] = nil }
+            @fields.each do |field|
+              if @if_equals.nil?
+                row[field] = nil
+              else
+                row[field] = nil if row[field] == @if_equals
+              end
+            end
             row
           end
         end

diff --git a/spec/kiba/extend/transforms/clean_spec.rb b/spec/kiba/extend/transforms/clean_spec.rb
@@ -42,13 +42,30 @@
         ['5', 'Person;notmapped']
       ]
 
-      before { generate_csv(test_csv, rows) }
-      let(:result) { execute_job(filename: test_csv,
-                                 xform: Clean::ClearFields,
-                                 xformopt: {fields: %i[type]}) }
+    before { generate_csv(test_csv, rows) }
+    context 'without additional arguments' do
       it 'sets the value of the field in all rows to nil' do
+        result = execute_job(filename: test_csv,
+                                   xform: Clean::ClearFields,
+                                   xformopt: {fields: %i[type]})
+        expect(result.map{ |e| e[1]}.uniq[0]).to be_nil
+      end
+    end
+    context 'with if_equals argument' do
+      it 'sets the value of the field to nil if it matches `if_equals`' do
+        result = execute_job(filename: test_csv,
+                             xform: Clean::ClearFields,
+                             xformopt: {fields: %i[type], if_equals: ';'})
+        expected = [
+          {:id=>'1', :type=>'Person;unmapped;Organization'},
+          {:id=>'2', :type=>nil},
+          {:id=>'3', :type=>nil},
+          {:id=>'4', :type=>''},
+          {:id=>'5', :type=>'Person;notmapped'},
+        ]
         expect(result.map{ |e| e[1]}.uniq[0]).to be_nil
       end
+    end
   end
 
   describe 'DelimiterOnlyFields' do