From a2856fae86884f704b47882f9b32d2fce539fcdb Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Fri, 1 Oct 2021 20:47:27 -0400 Subject: [PATCH 1/2] refactor tests --- spec/kiba/extend/transforms/append_spec.rb | 64 ++++----- spec/kiba/extend/transforms/clean_spec.rb | 154 ++++++++++++--------- 2 files changed, 121 insertions(+), 97 deletions(-) diff --git a/spec/kiba/extend/transforms/append_spec.rb b/spec/kiba/extend/transforms/append_spec.rb index 321816526..43a76dadf 100644 --- a/spec/kiba/extend/transforms/append_spec.rb +++ b/spec/kiba/extend/transforms/append_spec.rb @@ -3,51 +3,51 @@ require 'spec_helper' RSpec.describe Kiba::Extend::Transforms::Append do - before { generate_csv(rows) } + let(:accumulator){ [] } + let(:test_job){ Helpers::TestJob.new(input: input, accumulator: accumulator, transforms: transforms) } + let(:result){ test_job.accumulator } describe 'NilFields' do - let(:rows) do - [ - %w[id z], - [1, 'zz'] - ] - end - let(:result) do - execute_job(filename: test_csv, - xform: Append::NilFields, - xformopt: { fields: %i[a b c z] }) + let(:input) { [{ z: 'zz' }] } + + let(:transforms) do + Kiba.job_segment do + transform Append::NilFields, fields: %i[a b c z] + end end + + let(:expected) { [{ z: 'zz', a: nil, b: nil, c: nil }] } + it 'adds non-existing fields, populating with nil, while leaving existing fields alone' do - expected = { id: '1', z: 'zz', a: nil, b: nil, c: nil } - expect(result[0]).to eq(expected) + expect(result).to eq(expected) end end describe 'ToFieldValue' do - let(:rows) do + let(:input) do [ - %w[id name], - [1, 'Weddy'], - [2, nil], - [3, ''] + { name: 'Weddy' }, + { name: nil }, + { name: '' } ] end - let(:result) do - execute_job(filename: test_csv, - xform: Append::ToFieldValue, - xformopt: { field: :name, value: ' (name)' }) - end - it 'prepends given value to existing field values' do - expected = { id: '1', name: 'Weddy (name)' } - expect(result[0]).to eq(expected) + + let(:transforms) do + Kiba.job_segment do + transform Append::ToFieldValue, field: :name, value: ' (name)' + end end - it 'leaves nil values alone' do - expected = { id: '2', name: nil } - expect(result[1]).to eq(expected) + + let(:expected) do + [ + { name: 'Weddy (name)' }, + { name: nil }, + { name: '' } + ] end - it 'leaves blank values alone' do - expected = { id: '3', name: '' } - expect(result[2]).to eq(expected) + + it 'prepends given value to existing field values, leaving blank values alone' do + expect(result).to eq(expected) end end end diff --git a/spec/kiba/extend/transforms/clean_spec.rb b/spec/kiba/extend/transforms/clean_spec.rb index 96dbdde0c..5c15aa10c 100644 --- a/spec/kiba/extend/transforms/clean_spec.rb +++ b/spec/kiba/extend/transforms/clean_spec.rb @@ -3,88 +3,112 @@ require 'spec_helper' RSpec.describe Kiba::Extend::Transforms::Clean do - describe 'AlphabetizeFieldValues' do - # test_csv = File.join(__dir__, 'tmp', 'test.csv') - # binding.pry - # - rows = [ - %w[type], - ['Person;unmapped;Organization'], - [';'], - [nil], - [''], - ['Person;notmapped'], - ['%NULLVALUE%;apple'], - ['oatmeal;%NULLVALUE%'] - ] + let(:accumulator){ [] } + let(:test_job){ Helpers::TestJob.new(input: input, accumulator: accumulator, transforms: transforms) } + let(:result){ test_job.accumulator } - before { generate_csv(rows) } - let(:usenull) { false } - let(:direction) { :asc } - let(:result) do - execute_job(filename: test_csv, - xform: Clean::AlphabetizeFieldValues, - xformopt: { fields: %i[type], delim: ';', usenull: usenull, direction: direction }) + describe 'AlphabetizeFieldValues' do + let(:input) do + [ + { type: 'Person;unmapped;Organization'}, + { type: ';'}, + { type: nil}, + { type: ''}, + { type: 'Person;notmapped'}, + { type: '%NULLVALUE%;apple'}, + { type: 'oatmeal;%NULLVALUE%'} + ] end - context 'when usenull = false' do - it 'sorts as expected' do - expected = [ - 'Organization;Person;unmapped', - ';', - nil, - '', - 'notmapped;Person', - 'apple;%NULLVALUE%', - '%NULLVALUE%;oatmeal' - ] - expect(result.map { |res| res[:type] }).to eq(expected) - end - context 'when direction = :desc' do - let(:direction) { :desc } + context 'when when usenull = false' do + context 'when direction = :asc' do + let(:transforms) do + Kiba.job_segment do + transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false, + direction: :asc + end + end + it 'sorts as expected' do expected = [ - 'unmapped;Person;Organization', - ';', - nil, - '', - 'Person;notmapped', - '%NULLVALUE%;apple', - 'oatmeal;%NULLVALUE%' + { type: 'Organization;Person;unmapped'}, + { type: ';'}, + { type: nil}, + { type: ''}, + { type: 'notmapped;Person'}, + { type: 'apple;%NULLVALUE%'}, + { type: '%NULLVALUE%;oatmeal'} ] - expect(result.map { |res| res[:type] }).to eq(expected) + expect(result).to eq(expected) + end + + context 'when direction = :desc' do + let(:transforms) do + Kiba.job_segment do + transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false, + direction: :desc + end + end + + it 'sorts as expected' do + expected = [ + { type: 'unmapped;Person;Organization'}, + { type: ';'}, + { type: nil}, + { type: ''}, + { type: 'Person;notmapped'}, + { type: '%NULLVALUE%;apple'}, + { type: 'oatmeal;%NULLVALUE%'} + ] + expect(result).to eq(expected) + end end end end context 'when usenull = true' do - let(:usenull) { true } - it 'sorts as expected' do - expected = [ - 'Organization;Person;unmapped', - ';', - nil, - '', - 'notmapped;Person', - 'apple;%NULLVALUE%', - 'oatmeal;%NULLVALUE%' - ] - expect(result.map { |res| res[:type] }).to eq(expected) + context 'when direction = :asc' do + let(:transforms) do + Kiba.job_segment do + transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true, + direction: :asc + end + end + + it 'sorts as expected' do + expected = [ + { type: 'Organization;Person;unmapped'}, + { type: ';'}, + { type: nil}, + { type: ''}, + { type: 'notmapped;Person'}, + { type: 'apple;%NULLVALUE%'}, + { type: 'oatmeal;%NULLVALUE%'} + ] + expect(result).to eq(expected) + end end + context 'when direction = :desc' do - let(:direction) { :desc } + let(:transforms) do + Kiba.job_segment do + transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true, + direction: :desc + end + end + it 'sorts as expected' do expected = [ - 'unmapped;Person;Organization', - ';', - nil, - '', - 'Person;notmapped', - '%NULLVALUE%;apple', - '%NULLVALUE%;oatmeal' + { type: 'unmapped;Person;Organization'}, + { type: ';'}, + { type: nil}, + { type: ''}, + { type: 'Person;notmapped'}, + { type: '%NULLVALUE%;apple'}, + { type: '%NULLVALUE%;oatmeal'} ] - expect(result.map { |res| res[:type] }).to eq(expected) + expect(result).to eq(expected) end end end From 77ac67992ef8cc46ce2d04dda9069ed2a4d575b0 Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Fri, 1 Oct 2021 20:47:44 -0400 Subject: [PATCH 2/2] document transforms --- lib/kiba/extend/transforms/append.rb | 56 ++++++++++++ lib/kiba/extend/transforms/clean.rb | 124 ++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 11 deletions(-) diff --git a/lib/kiba/extend/transforms/append.rb b/lib/kiba/extend/transforms/append.rb index 99fbb8995..fd1995bed 100644 --- a/lib/kiba/extend/transforms/append.rb +++ b/lib/kiba/extend/transforms/append.rb @@ -6,6 +6,32 @@ module Transforms # Adds values to the end of fields or rows module Append ::Append = Kiba::Extend::Transforms::Append + + # Adds the given field(s) to the row with nil value if they do not already exist in row + # + # # Examples + # + # Input table: + # + # ``` + # | z | + # |----| + # | zz | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Append::NilFields, fields: %i[a b c z] + # ``` + # + # Results in: + # + # ``` + # | z | a | b | c | + # |----+-----+-----+-----| + # | zz | nil | nil | nil | + # ``` class NilFields def initialize(fields:) @fields = [fields].flatten @@ -20,6 +46,36 @@ def process(row) end end + # Adds the given value to the end of value of the given field. Does not affect nil/empty field values + # + # # Examples + # + # Input table: + # + # ``` + # ``` + # | name | + # |-------| + # | Weddy | + # | nil | + # | | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Append::ToFieldValue, field: :name, value: ' (name)' + # ``` + # + # Results in: + # + # ``` + # | name | + # |--------------| + # | Weddy (name) | + # | nil | + # | | + # ``` class ToFieldValue def initialize(field:, value:) @field = field diff --git a/lib/kiba/extend/transforms/clean.rb b/lib/kiba/extend/transforms/clean.rb index 53cf2261e..ae1833172 100644 --- a/lib/kiba/extend/transforms/clean.rb +++ b/lib/kiba/extend/transforms/clean.rb @@ -9,15 +9,117 @@ module Transforms module Clean ::Clean = Kiba::Extend::Transforms::Clean + # Sorts the multiple values within a field alphabetically + # # @note This transformation does **NOT** sort the **ROWS** in a dataset. It sorts values within # individual fields of a row - # Sorts the multiple values within a field alphabetically - # @param fields [Array(Symbol)] names of fields to sort - # @param delim [String] Character(s) on which to split field values - # @param usenull [Boolean] Whether to treat %NULLVALUE% as a blank in processing - # @param direction [:asc, :desc] Direction in which to sort field values + # + # # Examples + # + # Input table: + # + # ``` + # | type | + # |------------------------------| + # | Person;unmapped;Organization | + # | ; | + # | nil | + # | | + # | Person;notmapped | + # | %NULLVALUE%;apple | + # | oatmeal;%NULLVALUE% | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false, + # direction: :asc + # ``` + # + # Results in: + # + # ``` + # | type | + # |------------------------------| + # | Organization;Person;unmapped | + # | ; | + # | nil | + # | | + # | notmapped;Person | + # | apple;%NULLVALUE% | + # | %NULLVALUE%;oatmeal | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false, + # direction: :desc + # ``` + # + # Results in: + # + # ``` + # | type | + # |------------------------------| + # | unmapped;Person;Organization | + # | ; | + # | nil | + # | | + # | Person;notmapped | + # | %NULLVALUE%;apple | + # | oatmeal;%NULLVALUE% | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true, + # direction: :asc + # ``` + # + # Results in: + # + # ``` + # | type | + # |------------------------------| + # | Organization;Person;unmapped | + # | ; | + # | nil | + # | | + # | notmapped;Person | + # | apple;%NULLVALUE% | + # | oatmeal;%NULLVALUE% | + # ``` + # + # Used in pipeline as: + # + # ``` + # transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true, + # direction: :desc + # ``` + # + # Results in: + # + # ``` + # | type | + # |------------------------------| + # | unmapped;Person;Organization | + # | ; | + # | nil | + # | | + # | Person;notmapped | + # | %NULLVALUE%;apple | + # | %NULLVALUE%;oatmeal | + # ``` class AlphabetizeFieldValues include Kiba::Extend::Transforms::Helpers + + # @param fields [Array(Symbol)] names of fields to sort + # @param delim [String] Character(s) on which to split field values + # @param usenull [Boolean] Whether to treat %NULLVALUE% as a blank in processing + # @param direction [:asc, :desc] Direction in which to sort field values def initialize(fields:, delim:, usenull: false, direction: :asc) @fields = [fields].flatten @delim = delim @@ -127,7 +229,7 @@ def process_group(row, group) thisgroup.map! { |val| add_null_values(val) } if @use_nullvalue thisgroup.map! { |val| val.nil? ? [] : " #{val} ".split(@sep) } - .map! { |arr| arr.map(&:strip) } + .map! { |arr| arr.map(&:strip) } cts = thisgroup.map(&:size).uniq.reject(&:zero?) @@ -157,15 +259,15 @@ def add_null_values(str) return str if str.nil? str.sub(/^#{@sep}/, "%NULLVALUE%#{@sep}") - .sub(/#{@sep}$/, "#{@sep}%NULLVALUE%") - .gsub(/#{@sep}#{@sep}/, "#{@sep}%NULLVALUE%#{@sep}") + .sub(/#{@sep}$/, "#{@sep}%NULLVALUE%") + .gsub(/#{@sep}#{@sep}/, "#{@sep}%NULLVALUE%#{@sep}") end def all_empty?(group, index) thesevals = group.map { |arr| arr[index] } - .map { |val| empty_val(val) ? nil : val } - .uniq - .compact + .map { |val| empty_val(val) ? nil : val } + .uniq + .compact thesevals.empty? ? true : false end end