From 8bce6b6a15ebdbb2bbb372a8ea66dad31ecc934e Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Tue, 1 Jun 2021 12:38:44 -0400 Subject: [PATCH 1/3] bump kiba-extend version --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c5d2f841a..9374a37fe 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kiba-extend (1.12.0) + kiba-extend (1.12.1) activesupport kiba (>= 4.0.0) kiba-common (>= 1.5.0) From 4fae2c36c6255aa0aec02f3e6aa13156b4269a4c Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Tue, 1 Jun 2021 12:39:00 -0400 Subject: [PATCH 2/3] add Merge::CompareFieldsFlag transform --- lib/kiba/extend/transforms/merge.rb | 23 +++++++++ lib/kiba/extend/version.rb | 2 +- spec/kiba/extend/transforms/merge_spec.rb | 60 +++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/lib/kiba/extend/transforms/merge.rb b/lib/kiba/extend/transforms/merge.rb index bcdc47f5c..3c63f6b4d 100644 --- a/lib/kiba/extend/transforms/merge.rb +++ b/lib/kiba/extend/transforms/merge.rb @@ -4,6 +4,29 @@ module Transforms module Merge ::Merge = Kiba::Extend::Transforms::Merge + class CompareFieldsFlag + def initialize(fields:, target:, downcase: true, strip: true) + @fields = fields + @target = target + @strip = strip + @downcase = downcase + end + + def process(row) + row[@target] = 'diff' + values = [] + @fields.each do |field| + value = row.fetch(field, '').dup + value = '' if value.nil? + value.downcase! if @downcase + value.strip! if @strip + values << value + end + row[@target] = 'same' if values.uniq.length == 1 + row + end + end + class ConstantValue def initialize(target:, value:) @target = target diff --git a/lib/kiba/extend/version.rb b/lib/kiba/extend/version.rb index 64a725c6d..782b32a59 100644 --- a/lib/kiba/extend/version.rb +++ b/lib/kiba/extend/version.rb @@ -1,5 +1,5 @@ module Kiba module Extend - VERSION = "1.12.1" + VERSION = "1.13.0" end end diff --git a/spec/kiba/extend/transforms/merge_spec.rb b/spec/kiba/extend/transforms/merge_spec.rb index d31eac359..3fc7e77a2 100644 --- a/spec/kiba/extend/transforms/merge_spec.rb +++ b/spec/kiba/extend/transforms/merge_spec.rb @@ -8,6 +8,66 @@ after do File.delete(test_csv) if File.exist?(test_csv) end + + describe 'CompareFieldsFlag' do + let(:rows) { [ + ['id', 'pid', 'zid'], + [1, 1, 1], + [2, nil, 2], + ['three', 'Three', 'three'], + ['three', ' three', 'three'], + [4, 4, 3] + ] } + + context 'with defaults (downcase and strip)' do + it 'merges "same" or "diff" into target field after comparing values in row fields' do + expected = [ + {id: '1', pid: '1', zid: '1', comp: 'same'}, + {id: '2', pid: nil, zid: '2', comp: 'diff'}, + {id: 'three', pid: 'Three', zid: 'three', comp: 'same'}, + {id: 'three', pid: ' three', zid: 'three', comp: 'same'}, + {id: '4', pid: '4', zid: '3', comp: 'diff'}, + ] + result = execute_job(filename: test_csv, + xform: Merge::CompareFieldsFlag, + xformopt: {fields: %i[id pid zid], target: :comp}) + expect(result).to eq(expected) + end + end + + context 'with downcase false' do + it 'merges "same" or "diff" into target field after comparing values in row fields' do + expected = [ + {id: '1', pid: '1', zid: '1', comp: 'same'}, + {id: '2', pid: nil, zid: '2', comp: 'diff'}, + {id: 'three', pid: 'Three', zid: 'three', comp: 'diff'}, + {id: 'three', pid: ' three', zid: 'three', comp: 'same'}, + {id: '4', pid: '4', zid: '3', comp: 'diff'}, + ] + result = execute_job(filename: test_csv, + xform: Merge::CompareFieldsFlag, + xformopt: {fields: %i[id pid zid], target: :comp, downcase: false}) + expect(result).to eq(expected) + end + end + + context 'with strip false)' do + it 'merges "same" or "diff" into target field after comparing values in row fields' do + expected = [ + {id: '1', pid: '1', zid: '1', comp: 'same'}, + {id: '2', pid: nil, zid: '2', comp: 'diff'}, + {id: 'three', pid: 'Three', zid: 'three', comp: 'same'}, + {id: 'three', pid: ' three', zid: 'three', comp: 'diff'}, + {id: '4', pid: '4', zid: '3', comp: 'diff'}, + ] + result = execute_job(filename: test_csv, + xform: Merge::CompareFieldsFlag, + xformopt: {fields: %i[id pid zid], target: :comp, strip: false}) + expect(result).to eq(expected) + end + end + end + describe 'ConstantValue' do let(:rows) { [ ['id', 'name', 'sex', 'source'], From 26553b5bf621dbf3e85f4350f3d4d5cc885cf31d Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Tue, 1 Jun 2021 12:50:48 -0400 Subject: [PATCH 3/3] add `ignore_blank` option to `Merge::CompareFieldsFlag` --- Gemfile.lock | 2 +- lib/kiba/extend/transforms/merge.rb | 6 +++-- spec/kiba/extend/transforms/merge_spec.rb | 27 ++++++++++++++++++++--- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9374a37fe..57b26282e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - kiba-extend (1.12.1) + kiba-extend (1.13.0) activesupport kiba (>= 4.0.0) kiba-common (>= 1.5.0) diff --git a/lib/kiba/extend/transforms/merge.rb b/lib/kiba/extend/transforms/merge.rb index 3c63f6b4d..321592715 100644 --- a/lib/kiba/extend/transforms/merge.rb +++ b/lib/kiba/extend/transforms/merge.rb @@ -5,11 +5,12 @@ module Merge ::Merge = Kiba::Extend::Transforms::Merge class CompareFieldsFlag - def initialize(fields:, target:, downcase: true, strip: true) + def initialize(fields:, target:, downcase: true, strip: true, ignore_blank: false) @fields = fields @target = target @strip = strip @downcase = downcase + @ignore_blank = ignore_blank end def process(row) @@ -22,7 +23,8 @@ def process(row) value.strip! if @strip values << value end - row[@target] = 'same' if values.uniq.length == 1 + values.reject!{ |val| val.blank? } if @ignore_blank + row[@target] = 'same' if values.uniq.length == 1 || values.empty? row end end diff --git a/spec/kiba/extend/transforms/merge_spec.rb b/spec/kiba/extend/transforms/merge_spec.rb index 3fc7e77a2..3e6ae3eb1 100644 --- a/spec/kiba/extend/transforms/merge_spec.rb +++ b/spec/kiba/extend/transforms/merge_spec.rb @@ -16,10 +16,11 @@ [2, nil, 2], ['three', 'Three', 'three'], ['three', ' three', 'three'], - [4, 4, 3] + [4, 4, 3], + ['', nil, ''] ] } - context 'with defaults (downcase and strip)' do + context 'with defaults (downcase and strip = true, ignore_blank = false)' do it 'merges "same" or "diff" into target field after comparing values in row fields' do expected = [ {id: '1', pid: '1', zid: '1', comp: 'same'}, @@ -27,6 +28,7 @@ {id: 'three', pid: 'Three', zid: 'three', comp: 'same'}, {id: 'three', pid: ' three', zid: 'three', comp: 'same'}, {id: '4', pid: '4', zid: '3', comp: 'diff'}, + {id: '', pid: nil, zid: '', comp: 'same'} ] result = execute_job(filename: test_csv, xform: Merge::CompareFieldsFlag, @@ -43,6 +45,7 @@ {id: 'three', pid: 'Three', zid: 'three', comp: 'diff'}, {id: 'three', pid: ' three', zid: 'three', comp: 'same'}, {id: '4', pid: '4', zid: '3', comp: 'diff'}, + {id: '', pid: nil, zid: '', comp: 'same'} ] result = execute_job(filename: test_csv, xform: Merge::CompareFieldsFlag, @@ -51,7 +54,7 @@ end end - context 'with strip false)' do + context 'with strip false' do it 'merges "same" or "diff" into target field after comparing values in row fields' do expected = [ {id: '1', pid: '1', zid: '1', comp: 'same'}, @@ -59,6 +62,7 @@ {id: 'three', pid: 'Three', zid: 'three', comp: 'same'}, {id: 'three', pid: ' three', zid: 'three', comp: 'diff'}, {id: '4', pid: '4', zid: '3', comp: 'diff'}, + {id: '', pid: nil, zid: '', comp: 'same'} ] result = execute_job(filename: test_csv, xform: Merge::CompareFieldsFlag, @@ -66,6 +70,23 @@ expect(result).to eq(expected) end end + + context 'with ignore_blank = true' do + it 'merges "same" or "diff" into target field after comparing values in row fields' do + expected = [ + {id: '1', pid: '1', zid: '1', comp: 'same'}, + {id: '2', pid: nil, zid: '2', comp: 'same'}, + {id: 'three', pid: 'Three', zid: 'three', comp: 'same'}, + {id: 'three', pid: ' three', zid: 'three', comp: 'same'}, + {id: '4', pid: '4', zid: '3', comp: 'diff'}, + {id: '', pid: nil, zid: '', comp: 'same'} + ] + result = execute_job(filename: test_csv, + xform: Merge::CompareFieldsFlag, + xformopt: {fields: %i[id pid zid], target: :comp, ignore_blank: true}) + expect(result).to eq(expected) + end + end end describe 'ConstantValue' do