Merge pull request #8 from lyrasis/fields-to-rows

add Explode::FieldValuesToNewRows
lyrasis · Nov 19, 2020 · 059d47f · 059d47f
2 parents 51a0ab6 + 2757cc4
commit 059d47f
Show file tree

Hide file tree

Showing 3 changed files with 216 additions and 10 deletions.
diff --git a/lib/kiba/extend/transforms/explode.rb b/lib/kiba/extend/transforms/explode.rb
@@ -62,6 +62,47 @@ def process(row)
           end
         end
 
+        class FieldValuesToNewRows
+          def initialize(fields: [], target:, multival: false, sep: ' ', keep_nil: false, keep_empty: false)
+            @fields = fields
+            @target = target
+            @multival = multival
+            @sep = sep
+            @keep_nil = keep_nil
+            @keep_empty = keep_empty
+          end
+
+          def process(row)
+            rows = []
+            other_fields = row.keys - @fields
+            other_data = {}
+            other_fields.each{ |f| other_data[f] = row.fetch(f, nil) }
+
+            @fields.each do |field|
+              val = row.fetch(field, nil)
+              if val.nil?
+                vals = [nil]
+              elsif val.empty?
+                vals = ['']
+              elsif @multival
+                vals = val.split(@sep, -1)
+              else
+                vals = [val]
+              end
+
+              vals.each do |val|
+                next if val.nil? unless @keep_nil
+                next if val.empty? unless val.nil? || @keep_empty
+                new_row = other_data.clone
+                new_row[@target] = val
+                rows << new_row
+              end
+            end
+            rows.each{ |r| yield(r) }
+            nil
+          end
+        end
+
       end # module Explode
     end #module Transforms
   end #module Extend

diff --git a/lib/kiba/extend/version.rb b/lib/kiba/extend/version.rb
@@ -1,5 +1,5 @@
 module Kiba
   module Extend
-    VERSION = "1.5.0"
+    VERSION = "1.6.0"
   end
 end
diff --git a/spec/kiba/extend/transforms/explode_spec.rb b/spec/kiba/extend/transforms/explode_spec.rb
@@ -11,12 +11,12 @@
     before do
       generate_csv(test_csv, rows)
     end
-  after do
-    File.delete(test_csv) if File.exist?(test_csv)
-    File.delete('tmp/lkp.csv') if File.exist?('tmp/lkp.csv')
-  end
+    after do
+      File.delete(test_csv) if File.exist?(test_csv)
+      File.delete('tmp/lkp.csv') if File.exist?('tmp/lkp.csv')
+    end
 
-  context 'when given field :r1 and delim \';\'' do
+    context 'when given field :r1 and delim \';\'' do
       it 'creates 2 rows with same :id and :r2 fields' do
         expected = [
           { id: '001', r1: 'a', r2: 'foo;bar' },
@@ -34,7 +34,7 @@
   describe 'ColumnsRemappedInNewRows' do
     test_csv = 'tmp/test.csv'
     rows = [
-      ['id', 'r1', 'r2', 'ra', 'rb', 'xq'],
+      ['id',  'r1',  'r2',      'ra', 'rb', 'xq'],
       ['001', 'a;b', 'foo;bar', 'aa', 'bb', 'eee']
     ]
 
@@ -55,15 +55,180 @@
         result = execute_job(filename: test_csv,
                              xform: Explode::ColumnsRemappedInNewRows,
                              xformopt: { remap_groups: [
-                                                         %i[r1 r2],
-                                                         %i[ra rb]
-                                                       ],
+                               %i[r1 r2],
+                               %i[ra rb]
+                             ],
                                         map_to: %i[a1 a2]
                                        }
                             )
         expect(result).to eq(expected)
       end
     end
   end
+
+  describe 'FieldValuesToNewRows' do
+    test_csv = 'tmp/test.csv'
+    rows = [
+      ['id', 'child', 'parent'],
+      [1, 'a;b', 'c;d'],
+      [2, 'a', 'b'],
+      [3, '', 'q'],
+      [4, 'n', nil],
+      [5, '', nil],
+      [6, 'p;', ';z'],
+      [7, 'm;;n', 's']
+    ]
+    before do
+      generate_csv(test_csv, rows)
+    end
+
+    context 'when multival = true' do
+      context 'when keep_nil and keep_empty= false' do
+        it 'reshapes the columns as specified' do
+          expected = [
+            { id: '1', val: 'a' },
+            { id: '1', val: 'b' },
+            { id: '1', val: 'c' },
+            { id: '1', val: 'd' },
+            { id: '2', val: 'a' },
+            { id: '2', val: 'b' },
+            { id: '3', val: 'q' },
+            { id: '4', val: 'n' },
+            { id: '6', val: 'p' },
+            { id: '6', val: 'z' },
+            { id: '7', val: 'm' },
+            { id: '7', val: 'n' },
+            { id: '7', val: 's' }
+          ]
+          result = execute_job(filename: test_csv,
+                               xform: Explode::FieldValuesToNewRows,
+                               xformopt: {fields: %i[child parent],
+                                          target: :val,
+                                          multival: true,
+                                          sep: ';'
+                                         })
+          expect(result).to eq(expected)
+        end
+      end
+     context 'when keep_nil = true' do
+        it 'reshapes the columns as specified' do
+          expected = [
+            { id: '1', val: 'a' },
+            { id: '1', val: 'b' },
+            { id: '1', val: 'c' },
+            { id: '1', val: 'd' },
+            { id: '2', val: 'a' },
+            { id: '2', val: 'b' },
+            { id: '3', val: 'q' },
+            { id: '4', val: 'n' },
+            { id: '4', val: nil },
+            { id: '5', val: nil },
+            { id: '6', val: 'p' },
+            { id: '6', val: 'z' },
+            { id: '7', val: 'm' },
+            { id: '7', val: 'n' },
+            { id: '7', val: 's' },
+          ]
+          result = execute_job(filename: test_csv,
+                               xform: Explode::FieldValuesToNewRows,
+                               xformopt: {fields: %i[child parent],
+                                          target: :val,
+                                          multival: true,
+                                          sep: ';',
+                                          keep_nil: true
+                                         })
+          expect(result).to eq(expected)
+        end
+     end
+
+     context 'when keep_empty = true' do
+       it 'reshapes the columns as specified' do
+         expected = [
+           { id: '1', val: 'a' },
+           { id: '1', val: 'b' },
+           { id: '1', val: 'c' },
+           { id: '1', val: 'd' },
+           { id: '2', val: 'a' },
+           { id: '2', val: 'b' },
+           { id: '3', val: '' },
+           { id: '3', val: 'q' },
+           { id: '4', val: 'n' },
+           { id: '5', val: '' },
+           { id: '6', val: 'p' },
+           { id: '6', val: '' },
+           { id: '6', val: '' },
+           { id: '6', val: 'z' },
+           { id: '7', val: 'm' },
+           { id: '7', val: '' },
+           { id: '7', val: 'n' },
+           { id: '7', val: 's' },
+         ]
+         result = execute_job(filename: test_csv,
+                              xform: Explode::FieldValuesToNewRows,
+                              xformopt: {fields: %i[child parent],
+                                         target: :val,
+                                         multival: true,
+                                         sep: ';',
+                                         keep_empty: true
+                                        })
+         expect(result).to eq(expected)
+       end
+     end
+    end
+
+    context 'when multival = false' do
+      context 'when keep_nil and keep_empty = false' do
+        it 'reshapes the columns as specified' do
+          expected = [
+            { id: '1', val: 'a;b' },
+            { id: '1', val: 'c;d' },
+            { id: '2', val: 'a' },
+            { id: '2', val: 'b' },
+            { id: '3', val: 'q' },
+            { id: '4', val: 'n' },
+            { id: '6', val: 'p;' },
+            { id: '6', val: ';z' },
+            { id: '7', val: 'm;;n' },
+            { id: '7', val: 's' },
+          ]
+          result = execute_job(filename: test_csv,
+                               xform: Explode::FieldValuesToNewRows,
+                               xformopt: {fields: %i[child parent],
+                                          target: :val
+                                         })
+          expect(result).to eq(expected)
+        end
+      end
+
+      context 'when keep_nil and keep_empty = true' do
+        it 'reshapes the columns as specified' do
+          expected = [
+            { id: '1', val: 'a;b' },
+            { id: '1', val: 'c;d' },
+            { id: '2', val: 'a' },
+            { id: '2', val: 'b' },
+            { id: '3', val: '' },            
+            { id: '3', val: 'q' },
+            { id: '4', val: 'n' },
+            { id: '4', val: nil },
+            { id: '5', val: '' },
+            { id: '5', val: nil },
+            { id: '6', val: 'p;' },
+            { id: '6', val: ';z' },
+            { id: '7', val: 'm;;n' },
+            { id: '7', val: 's' },
+          ]
+          result = execute_job(filename: test_csv,
+                               xform: Explode::FieldValuesToNewRows,
+                               xformopt: {fields: %i[child parent],
+                                          target: :val,
+                                          keep_nil: true,
+                                          keep_empty: true
+                                         })
+          expect(result).to eq(expected)
+        end
+      end
+    end
+  end
 end