Skip to content

Commit

Permalink
Merge pull request #46 from lyrasis/refactor-tests-and-add-docs
Browse files Browse the repository at this point in the history 
Refactor tests and add docs
  • Loading branch information
@kspurgin
kspurgin authored Oct 2, 2021
2 parents a895e5f + 77ac679 commit 216ac1a
Show file tree
Hide file tree
Showing 4 changed files with 290 additions and 108 deletions.
56 changes: 56 additions & 0 deletions lib/kiba/extend/transforms/append.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,32 @@ module Transforms
# Adds values to the end of fields or rows
module Append
::Append = Kiba::Extend::Transforms::Append

# Adds the given field(s) to the row with nil value if they do not already exist in row
#
# # Examples
#
# Input table:
#
# ```
# | z |
# |----|
# | zz |
# ```
#
# Used in pipeline as:
#
# ```
# transform Append::NilFields, fields: %i[a b c z]
# ```
#
# Results in:
#
# ```
# | z | a | b | c |
# |----+-----+-----+-----|
# | zz | nil | nil | nil |
# ```
class NilFields
def initialize(fields:)
@fields = [fields].flatten
Expand All @@ -20,6 +46,36 @@ def process(row)
end
end

# Adds the given value to the end of value of the given field. Does not affect nil/empty field values
#
# # Examples
#
# Input table:
#
# ```
# ```
# | name |
# |-------|
# | Weddy |
# | nil |
# | |
# ```
#
# Used in pipeline as:
#
# ```
# transform Append::ToFieldValue, field: :name, value: ' (name)'
# ```
#
# Results in:
#
# ```
# | name |
# |--------------|
# | Weddy (name) |
# | nil |
# | |
# ```
class ToFieldValue
def initialize(field:, value:)
@field = field
Expand Down
124 changes: 113 additions & 11 deletions lib/kiba/extend/transforms/clean.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,117 @@ module Transforms
module Clean
::Clean = Kiba::Extend::Transforms::Clean

# Sorts the multiple values within a field alphabetically
#
# @note This transformation does **NOT** sort the **ROWS** in a dataset. It sorts values within
# individual fields of a row
# Sorts the multiple values within a field alphabetically
# @param fields [Array(Symbol)] names of fields to sort
# @param delim [String] Character(s) on which to split field values
# @param usenull [Boolean] Whether to treat %NULLVALUE% as a blank in processing
# @param direction [:asc, :desc] Direction in which to sort field values
#
# # Examples
#
# Input table:
#
# ```
# | type |
# |------------------------------|
# | Person;unmapped;Organization |
# | ; |
# | nil |
# | |
# | Person;notmapped |
# | %NULLVALUE%;apple |
# | oatmeal;%NULLVALUE% |
# ```
#
# Used in pipeline as:
#
# ```
# transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false,
# direction: :asc
# ```
#
# Results in:
#
# ```
# | type |
# |------------------------------|
# | Organization;Person;unmapped |
# | ; |
# | nil |
# | |
# | notmapped;Person |
# | apple;%NULLVALUE% |
# | %NULLVALUE%;oatmeal |
# ```
#
# Used in pipeline as:
#
# ```
# transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: false,
# direction: :desc
# ```
#
# Results in:
#
# ```
# | type |
# |------------------------------|
# | unmapped;Person;Organization |
# | ; |
# | nil |
# | |
# | Person;notmapped |
# | %NULLVALUE%;apple |
# | oatmeal;%NULLVALUE% |
# ```
#
# Used in pipeline as:
#
# ```
# transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true,
# direction: :asc
# ```
#
# Results in:
#
# ```
# | type |
# |------------------------------|
# | Organization;Person;unmapped |
# | ; |
# | nil |
# | |
# | notmapped;Person |
# | apple;%NULLVALUE% |
# | oatmeal;%NULLVALUE% |
# ```
#
# Used in pipeline as:
#
# ```
# transform Clean::AlphabetizeFieldValues, fields: %i[type], delim: ';', usenull: true,
# direction: :desc
# ```
#
# Results in:
#
# ```
# | type |
# |------------------------------|
# | unmapped;Person;Organization |
# | ; |
# | nil |
# | |
# | Person;notmapped |
# | %NULLVALUE%;apple |
# | %NULLVALUE%;oatmeal |
# ```
class AlphabetizeFieldValues
include Kiba::Extend::Transforms::Helpers

# @param fields [Array(Symbol)] names of fields to sort
# @param delim [String] Character(s) on which to split field values
# @param usenull [Boolean] Whether to treat %NULLVALUE% as a blank in processing
# @param direction [:asc, :desc] Direction in which to sort field values
def initialize(fields:, delim:, usenull: false, direction: :asc)
@fields = [fields].flatten
@delim = delim
Expand Down Expand Up @@ -127,7 +229,7 @@ def process_group(row, group)
thisgroup.map! { |val| add_null_values(val) } if @use_nullvalue

thisgroup.map! { |val| val.nil? ? [] : " #{val} ".split(@sep) }
.map! { |arr| arr.map(&:strip) }
.map! { |arr| arr.map(&:strip) }

cts = thisgroup.map(&:size).uniq.reject(&:zero?)

Expand Down Expand Up @@ -157,15 +259,15 @@ def add_null_values(str)
return str if str.nil?

str.sub(/^#{@sep}/, "%NULLVALUE%#{@sep}")
.sub(/#{@sep}$/, "#{@sep}%NULLVALUE%")
.gsub(/#{@sep}#{@sep}/, "#{@sep}%NULLVALUE%#{@sep}")
.sub(/#{@sep}$/, "#{@sep}%NULLVALUE%")
.gsub(/#{@sep}#{@sep}/, "#{@sep}%NULLVALUE%#{@sep}")
end

def all_empty?(group, index)
thesevals = group.map { |arr| arr[index] }
.map { |val| empty_val(val) ? nil : val }
.uniq
.compact
.map { |val| empty_val(val) ? nil : val }
.uniq
.compact
thesevals.empty? ? true : false
end
end
Expand Down
64 changes: 32 additions & 32 deletions spec/kiba/extend/transforms/append_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,51 +3,51 @@
require 'spec_helper'

RSpec.describe Kiba::Extend::Transforms::Append do
before { generate_csv(rows) }
let(:accumulator){ [] }
let(:test_job){ Helpers::TestJob.new(input: input, accumulator: accumulator, transforms: transforms) }
let(:result){ test_job.accumulator }

describe 'NilFields' do
let(:rows) do
[
%w[id z],
[1, 'zz']
]
end
let(:result) do
execute_job(filename: test_csv,
xform: Append::NilFields,
xformopt: { fields: %i[a b c z] })
let(:input) { [{ z: 'zz' }] }

let(:transforms) do
Kiba.job_segment do
transform Append::NilFields, fields: %i[a b c z]
end
end

let(:expected) { [{ z: 'zz', a: nil, b: nil, c: nil }] }

it 'adds non-existing fields, populating with nil, while leaving existing fields alone' do
expected = { id: '1', z: 'zz', a: nil, b: nil, c: nil }
expect(result[0]).to eq(expected)
expect(result).to eq(expected)
end
end

describe 'ToFieldValue' do
let(:rows) do
let(:input) do
[
%w[id name],
[1, 'Weddy'],
[2, nil],
[3, '']
{ name: 'Weddy' },
{ name: nil },
{ name: '' }
]
end
let(:result) do
execute_job(filename: test_csv,
xform: Append::ToFieldValue,
xformopt: { field: :name, value: ' (name)' })
end
it 'prepends given value to existing field values' do
expected = { id: '1', name: 'Weddy (name)' }
expect(result[0]).to eq(expected)

let(:transforms) do
Kiba.job_segment do
transform Append::ToFieldValue, field: :name, value: ' (name)'
end
end
it 'leaves nil values alone' do
expected = { id: '2', name: nil }
expect(result[1]).to eq(expected)

let(:expected) do
[
{ name: 'Weddy (name)' },
{ name: nil },
{ name: '' }
]
end
it 'leaves blank values alone' do
expected = { id: '3', name: '' }
expect(result[2]).to eq(expected)

it 'prepends given value to existing field values, leaving blank values alone' do
expect(result).to eq(expected)
end
end
end
Loading

0 comments on commit 216ac1a

Please sign in to comment.