diff --git a/docs/Kiba/Extend/Transforms/Clean/ClearFields.html b/docs/Kiba/Extend/Transforms/Clean/ClearFields.html
index b27166702..8dbc9775d 100644
--- a/docs/Kiba/Extend/Transforms/Clean/ClearFields.html
+++ b/docs/Kiba/Extend/Transforms/Clean/ClearFields.html
@@ -172,13 +172,13 @@
-41
-42
-43
-44
+50
+51
+52
+53
- # File 'lib/kiba/extend/transforms/clean.rb', line 41
+ # File 'lib/kiba/extend/transforms/clean.rb', line 50
def initialize ( fields: , if_equals: nil )
@fields = fields
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Clean/DelimiterOnlyFields.html b/docs/Kiba/Extend/Transforms/Clean/DelimiterOnlyFields.html
index eca13cded..c7feabd61 100644
--- a/docs/Kiba/Extend/Transforms/Clean/DelimiterOnlyFields.html
+++ b/docs/Kiba/Extend/Transforms/Clean/DelimiterOnlyFields.html
@@ -89,7 +89,7 @@
Includes:
- Helpers Helpers
@@ -157,8 +157,8 @@
- Methods included from Helpers
- #delim_only?
+ Methods included from Helpers
+ #delim_only? #field_values
Constructor Details
@@ -188,13 +188,13 @@
-61
-62
-63
-64
+70
+71
+72
+73
- # File 'lib/kiba/extend/transforms/clean.rb', line 61
+ # File 'lib/kiba/extend/transforms/clean.rb', line 70
def initialize ( delim: , use_nullvalue: false )
@delim = delim
@@ -211,7 +211,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Clean/DowncaseFieldValues.html b/docs/Kiba/Extend/Transforms/Clean/DowncaseFieldValues.html
index 3f580b987..386ed6644 100644
--- a/docs/Kiba/Extend/Transforms/Clean/DowncaseFieldValues.html
+++ b/docs/Kiba/Extend/Transforms/Clean/DowncaseFieldValues.html
@@ -172,12 +172,12 @@
-76
-77
-78
+85
+86
+87
- # File 'lib/kiba/extend/transforms/clean.rb', line 76
+ # File 'lib/kiba/extend/transforms/clean.rb', line 85
def initialize ( fields: )
@fields = fields
@@ -193,7 +193,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Clean/EmptyFieldGroups.html b/docs/Kiba/Extend/Transforms/Clean/EmptyFieldGroups.html
index 9dd79fadb..05fb7268d 100644
--- a/docs/Kiba/Extend/Transforms/Clean/EmptyFieldGroups.html
+++ b/docs/Kiba/Extend/Transforms/Clean/EmptyFieldGroups.html
@@ -172,14 +172,14 @@
-97
-98
-99
-100
-101
+106
+107
+108
+109
+110
- # File 'lib/kiba/extend/transforms/clean.rb', line 97
+ # File 'lib/kiba/extend/transforms/clean.rb', line 106
def initialize ( groups: , sep: , use_nullvalue: false )
@groups = groups
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Clean/RegexpFindReplaceFieldVals.html b/docs/Kiba/Extend/Transforms/Clean/RegexpFindReplaceFieldVals.html
index 82733a1ed..f2b013217 100644
--- a/docs/Kiba/Extend/Transforms/Clean/RegexpFindReplaceFieldVals.html
+++ b/docs/Kiba/Extend/Transforms/Clean/RegexpFindReplaceFieldVals.html
@@ -172,18 +172,18 @@
-161
-162
-163
-164
-165
-166
-167
-168
-169
+170
+171
+172
+173
+174
+175
+176
+177
+178
- # File 'lib/kiba/extend/transforms/clean.rb', line 161
+ # File 'lib/kiba/extend/transforms/clean.rb', line 170
def initialize ( fields: , find: , replace: , casesensitive: true , multival: false , sep: nil , debug: false )
@fields = fields
@@ -205,7 +205,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Clean/StripFields.html b/docs/Kiba/Extend/Transforms/Clean/StripFields.html
index dee3dcdfa..e0b817b55 100644
--- a/docs/Kiba/Extend/Transforms/Clean/StripFields.html
+++ b/docs/Kiba/Extend/Transforms/Clean/StripFields.html
@@ -172,12 +172,12 @@
-203
-204
-205
+212
+213
+214
- # File 'lib/kiba/extend/transforms/clean.rb', line 203
+ # File 'lib/kiba/extend/transforms/clean.rb', line 212
def initialize ( fields: )
@fields = fields
@@ -193,7 +193,7 @@
diff --git a/docs/Kiba/Extend/Transforms/CombineValues.html b/docs/Kiba/Extend/Transforms/CombineValues.html
index a994aab0f..6f6776eb4 100644
--- a/docs/Kiba/Extend/Transforms/CombineValues.html
+++ b/docs/Kiba/Extend/Transforms/CombineValues.html
@@ -105,7 +105,7 @@ Defined Under Namespace
diff --git a/docs/Kiba/Extend/Transforms/CombineValues/AcrossFieldGroup.html b/docs/Kiba/Extend/Transforms/CombineValues/AcrossFieldGroup.html
index f7c58cfa7..568b79739 100644
--- a/docs/Kiba/Extend/Transforms/CombineValues/AcrossFieldGroup.html
+++ b/docs/Kiba/Extend/Transforms/CombineValues/AcrossFieldGroup.html
@@ -341,7 +341,7 @@ More on fieldmap paramete
diff --git a/docs/Kiba/Extend/Transforms/CombineValues/FromFieldsWithDelimiter.html b/docs/Kiba/Extend/Transforms/CombineValues/FromFieldsWithDelimiter.html
index fda6c4d93..f3f547159 100644
--- a/docs/Kiba/Extend/Transforms/CombineValues/FromFieldsWithDelimiter.html
+++ b/docs/Kiba/Extend/Transforms/CombineValues/FromFieldsWithDelimiter.html
@@ -346,7 +346,7 @@
diff --git a/docs/Kiba/Extend/Transforms/CombineValues/FullRecord.html b/docs/Kiba/Extend/Transforms/CombineValues/FullRecord.html
index b82b7979a..01f62b07e 100644
--- a/docs/Kiba/Extend/Transforms/CombineValues/FullRecord.html
+++ b/docs/Kiba/Extend/Transforms/CombineValues/FullRecord.html
@@ -269,7 +269,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Copy.html b/docs/Kiba/Extend/Transforms/Copy.html
index 0f0d38f0a..9d15ac00e 100644
--- a/docs/Kiba/Extend/Transforms/Copy.html
+++ b/docs/Kiba/Extend/Transforms/Copy.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Copy/Field.html b/docs/Kiba/Extend/Transforms/Copy/Field.html
index cf8885892..057ec0b9b 100644
--- a/docs/Kiba/Extend/Transforms/Copy/Field.html
+++ b/docs/Kiba/Extend/Transforms/Copy/Field.html
@@ -250,7 +250,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Cspace.html b/docs/Kiba/Extend/Transforms/Cspace.html
index 3cfeb88a1..dc0704d46 100644
--- a/docs/Kiba/Extend/Transforms/Cspace.html
+++ b/docs/Kiba/Extend/Transforms/Cspace.html
@@ -146,7 +146,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Cspace/ConvertToID.html b/docs/Kiba/Extend/Transforms/Cspace/ConvertToID.html
index eaa86a102..5a700ec4b 100644
--- a/docs/Kiba/Extend/Transforms/Cspace/ConvertToID.html
+++ b/docs/Kiba/Extend/Transforms/Cspace/ConvertToID.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Cspace/FlagInvalidCharacters.html b/docs/Kiba/Extend/Transforms/Cspace/FlagInvalidCharacters.html
index 4d82af4bd..c65e44b94 100644
--- a/docs/Kiba/Extend/Transforms/Cspace/FlagInvalidCharacters.html
+++ b/docs/Kiba/Extend/Transforms/Cspace/FlagInvalidCharacters.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Cspace/NormalizeForID.html b/docs/Kiba/Extend/Transforms/Cspace/NormalizeForID.html
index f4829f8a1..174b9e1db 100644
--- a/docs/Kiba/Extend/Transforms/Cspace/NormalizeForID.html
+++ b/docs/Kiba/Extend/Transforms/Cspace/NormalizeForID.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Deduplicate.html b/docs/Kiba/Extend/Transforms/Deduplicate.html
index 5c58fdad0..79e271d2a 100644
--- a/docs/Kiba/Extend/Transforms/Deduplicate.html
+++ b/docs/Kiba/Extend/Transforms/Deduplicate.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Deduplicate/FieldValues.html b/docs/Kiba/Extend/Transforms/Deduplicate/FieldValues.html
index 94053a9c9..690b64f67 100644
--- a/docs/Kiba/Extend/Transforms/Deduplicate/FieldValues.html
+++ b/docs/Kiba/Extend/Transforms/Deduplicate/FieldValues.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Deduplicate/Fields.html b/docs/Kiba/Extend/Transforms/Deduplicate/Fields.html
index dc9bb2c7c..b3aa5fcc0 100644
--- a/docs/Kiba/Extend/Transforms/Deduplicate/Fields.html
+++ b/docs/Kiba/Extend/Transforms/Deduplicate/Fields.html
@@ -201,7 +201,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Deduplicate/Flag.html b/docs/Kiba/Extend/Transforms/Deduplicate/Flag.html
index b14034072..6b3b2e69f 100644
--- a/docs/Kiba/Extend/Transforms/Deduplicate/Flag.html
+++ b/docs/Kiba/Extend/Transforms/Deduplicate/Flag.html
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Deduplicate/GroupedFieldValues.html b/docs/Kiba/Extend/Transforms/Deduplicate/GroupedFieldValues.html
index 1c1c2473e..fd871076c 100644
--- a/docs/Kiba/Extend/Transforms/Deduplicate/GroupedFieldValues.html
+++ b/docs/Kiba/Extend/Transforms/Deduplicate/GroupedFieldValues.html
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete.html b/docs/Kiba/Extend/Transforms/Delete.html
index 836102f96..8db24353b 100644
--- a/docs/Kiba/Extend/Transforms/Delete.html
+++ b/docs/Kiba/Extend/Transforms/Delete.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Delete/EmptyFieldValues.html b/docs/Kiba/Extend/Transforms/Delete/EmptyFieldValues.html
index 04415c64d..8d66f0170 100644
--- a/docs/Kiba/Extend/Transforms/Delete/EmptyFieldValues.html
+++ b/docs/Kiba/Extend/Transforms/Delete/EmptyFieldValues.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete/FieldValueContainingString.html b/docs/Kiba/Extend/Transforms/Delete/FieldValueContainingString.html
index bb5ff5f42..bb67c5c53 100644
--- a/docs/Kiba/Extend/Transforms/Delete/FieldValueContainingString.html
+++ b/docs/Kiba/Extend/Transforms/Delete/FieldValueContainingString.html
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete/FieldValueIfEqualsOtherField.html b/docs/Kiba/Extend/Transforms/Delete/FieldValueIfEqualsOtherField.html
index ead8ae1fe..d6a36c11d 100644
--- a/docs/Kiba/Extend/Transforms/Delete/FieldValueIfEqualsOtherField.html
+++ b/docs/Kiba/Extend/Transforms/Delete/FieldValueIfEqualsOtherField.html
@@ -203,7 +203,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete/FieldValueMatchingRegexp.html b/docs/Kiba/Extend/Transforms/Delete/FieldValueMatchingRegexp.html
index 3fb78cff0..bcb41050b 100644
--- a/docs/Kiba/Extend/Transforms/Delete/FieldValueMatchingRegexp.html
+++ b/docs/Kiba/Extend/Transforms/Delete/FieldValueMatchingRegexp.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete/Fields.html b/docs/Kiba/Extend/Transforms/Delete/Fields.html
index ba8278537..675c67f91 100644
--- a/docs/Kiba/Extend/Transforms/Delete/Fields.html
+++ b/docs/Kiba/Extend/Transforms/Delete/Fields.html
@@ -193,7 +193,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Delete/FieldsExcept.html b/docs/Kiba/Extend/Transforms/Delete/FieldsExcept.html
index 3ac773ff4..43b3b1359 100644
--- a/docs/Kiba/Extend/Transforms/Delete/FieldsExcept.html
+++ b/docs/Kiba/Extend/Transforms/Delete/FieldsExcept.html
@@ -193,7 +193,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Explode.html b/docs/Kiba/Extend/Transforms/Explode.html
index da8e9cdbf..cf90c855f 100644
--- a/docs/Kiba/Extend/Transforms/Explode.html
+++ b/docs/Kiba/Extend/Transforms/Explode.html
@@ -118,7 +118,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Explode/ColumnsRemappedInNewRows.html b/docs/Kiba/Extend/Transforms/Explode/ColumnsRemappedInNewRows.html
index 9ecd8d9a1..9f8a9ad4b 100644
--- a/docs/Kiba/Extend/Transforms/Explode/ColumnsRemappedInNewRows.html
+++ b/docs/Kiba/Extend/Transforms/Explode/ColumnsRemappedInNewRows.html
@@ -273,7 +273,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Explode/FieldValuesToNewRows.html b/docs/Kiba/Extend/Transforms/Explode/FieldValuesToNewRows.html
index 057001cac..63f005a4a 100644
--- a/docs/Kiba/Extend/Transforms/Explode/FieldValuesToNewRows.html
+++ b/docs/Kiba/Extend/Transforms/Explode/FieldValuesToNewRows.html
@@ -315,7 +315,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Explode/RowsFromMultivalField.html b/docs/Kiba/Extend/Transforms/Explode/RowsFromMultivalField.html
index 5ac46a334..326ace8a4 100644
--- a/docs/Kiba/Extend/Transforms/Explode/RowsFromMultivalField.html
+++ b/docs/Kiba/Extend/Transforms/Explode/RowsFromMultivalField.html
@@ -290,7 +290,7 @@
diff --git a/docs/Kiba/Extend/Transforms/FilterRows.html b/docs/Kiba/Extend/Transforms/FilterRows.html
index a3a14e00f..553bbed01 100644
--- a/docs/Kiba/Extend/Transforms/FilterRows.html
+++ b/docs/Kiba/Extend/Transforms/FilterRows.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/FilterRows/FieldEqualTo.html b/docs/Kiba/Extend/Transforms/FilterRows/FieldEqualTo.html
index eee401953..e5f1059b4 100644
--- a/docs/Kiba/Extend/Transforms/FilterRows/FieldEqualTo.html
+++ b/docs/Kiba/Extend/Transforms/FilterRows/FieldEqualTo.html
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/FilterRows/FieldMatchRegexp.html b/docs/Kiba/Extend/Transforms/FilterRows/FieldMatchRegexp.html
index 704ec81f7..bffa4e699 100644
--- a/docs/Kiba/Extend/Transforms/FilterRows/FieldMatchRegexp.html
+++ b/docs/Kiba/Extend/Transforms/FilterRows/FieldMatchRegexp.html
@@ -208,7 +208,7 @@
diff --git a/docs/Kiba/Extend/Transforms/FilterRows/FieldPopulated.html b/docs/Kiba/Extend/Transforms/FilterRows/FieldPopulated.html
index ecee100ed..4715ba404 100644
--- a/docs/Kiba/Extend/Transforms/FilterRows/FieldPopulated.html
+++ b/docs/Kiba/Extend/Transforms/FilterRows/FieldPopulated.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/FilterRows/FieldValueGreaterThan.html b/docs/Kiba/Extend/Transforms/FilterRows/FieldValueGreaterThan.html
index f8b101861..e4d718d8c 100644
--- a/docs/Kiba/Extend/Transforms/FilterRows/FieldValueGreaterThan.html
+++ b/docs/Kiba/Extend/Transforms/FilterRows/FieldValueGreaterThan.html
@@ -197,7 +197,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Helpers.html b/docs/Kiba/Extend/Transforms/Helpers.html
new file mode 100644
index 000000000..b7a17990e
--- /dev/null
+++ b/docs/Kiba/Extend/Transforms/Helpers.html
@@ -0,0 +1,487 @@
+
+
+
+
+ Module: Kiba::Extend::Transforms::Helpers
+
+ — Kiba-Extend Documentation
+
+
+
+
+
+
+
Module: Kiba::Extend::Transforms::Helpers
+
+
+
+
+
+
+
Overview
+
+
+
utility functions across Transforms
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Instance Method Summary
+ collapse
+
+
+
+
+
+
+
+
Instance Method Details
+
+
+
+
+
+ #delim_only? (val, delim, usenull = false) ⇒ false , true
+
+
+
+
+
+
+
+
+
Indicates whether a field value is delimiter-only. If usenull is set to true, the %NULLVALUE% string is treated as empty in detecting delimiter-only-ness
+
+
+
+
+
+
+
+
+
+
+17
+18
+19
+20
+21
+22
+23
+24
+
+
+ # File 'lib/kiba/extend/transforms/helpers.rb', line 17
+
+def delim_only? ( val , delim , usenull = false )
+ return false if val . nil?
+ return false if val . strip . empty?
+
+ chk = val . gsub ( delim , ' ' ) . strip
+ chk = chk . gsub ( ' %NULLVALUE% ' , ' ' ) . strip if usenull
+ chk . empty?
+end
+
+
+
+
+
+
+
+ #field_values (row:, fields:, discard: %i[nil empty delim], delim: DELIM, usenull: false) ⇒ Hash{Symbol=>String,Nil}l
+
+
+
+
+
+
+
+
+
Returns of field data for fields that meet keep criteria.
+
+
+
+
+
+
+
+
+
+
+32
+33
+34
+35
+36
+37
+38
+39
+
+
+ # File 'lib/kiba/extend/transforms/helpers.rb', line 32
+
+def field_values ( row: , fields: , discard: %i[ nil empty delim ] , delim: DELIM , usenull: false )
+ field_vals = fields . map { | field | [ field , row . fetch ( field , nil ) ] } . to_h
+ return field_vals if discard . blank?
+
+ field_vals = field_vals . reject { | field , val | val . nil? } if discard . any? ( :nil )
+ keep = keep_fields ( field_vals , discard , delim , usenull )
+ field_vals . select { | field , val | keep . any? ( field ) }
+end
+
+
+
+
+
+
+
+
+
+
+
Overview
diff --git a/docs/Kiba/Extend/Transforms/Merge/CompareFieldsFlag.html b/docs/Kiba/Extend/Transforms/Merge/CompareFieldsFlag.html
index c9c46b9f0..df341a59f 100644
--- a/docs/Kiba/Extend/Transforms/Merge/CompareFieldsFlag.html
+++ b/docs/Kiba/Extend/Transforms/Merge/CompareFieldsFlag.html
@@ -201,7 +201,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Merge/ConstantValue.html b/docs/Kiba/Extend/Transforms/Merge/ConstantValue.html
index 77ee69d01..6fb9f14e5 100644
--- a/docs/Kiba/Extend/Transforms/Merge/ConstantValue.html
+++ b/docs/Kiba/Extend/Transforms/Merge/ConstantValue.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Merge/ConstantValueConditional.html b/docs/Kiba/Extend/Transforms/Merge/ConstantValueConditional.html
index ff07b0e00..677b4b3e1 100644
--- a/docs/Kiba/Extend/Transforms/Merge/ConstantValueConditional.html
+++ b/docs/Kiba/Extend/Transforms/Merge/ConstantValueConditional.html
@@ -208,7 +208,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Merge/CountOfMatchingRows.html b/docs/Kiba/Extend/Transforms/Merge/CountOfMatchingRows.html
index 0a19ea90e..fc87ffdca 100644
--- a/docs/Kiba/Extend/Transforms/Merge/CountOfMatchingRows.html
+++ b/docs/Kiba/Extend/Transforms/Merge/CountOfMatchingRows.html
@@ -201,7 +201,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Merge/MultiRowLookup.html b/docs/Kiba/Extend/Transforms/Merge/MultiRowLookup.html
index 4ef4ec959..099fb4139 100644
--- a/docs/Kiba/Extend/Transforms/Merge/MultiRowLookup.html
+++ b/docs/Kiba/Extend/Transforms/Merge/MultiRowLookup.html
@@ -218,7 +218,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Merge/MultivalueConstant.html b/docs/Kiba/Extend/Transforms/Merge/MultivalueConstant.html
index 87f609963..3a8142249 100644
--- a/docs/Kiba/Extend/Transforms/Merge/MultivalueConstant.html
+++ b/docs/Kiba/Extend/Transforms/Merge/MultivalueConstant.html
@@ -201,7 +201,7 @@
diff --git a/docs/Kiba/Extend/Transforms/MsAccess.html b/docs/Kiba/Extend/Transforms/MsAccess.html
index 5d34a8286..9c42c7ba8 100644
--- a/docs/Kiba/Extend/Transforms/MsAccess.html
+++ b/docs/Kiba/Extend/Transforms/MsAccess.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/MsAccess/ScientificNotationToNumberString.html b/docs/Kiba/Extend/Transforms/MsAccess/ScientificNotationToNumberString.html
index afd7ff29a..94943f4d2 100644
--- a/docs/Kiba/Extend/Transforms/MsAccess/ScientificNotationToNumberString.html
+++ b/docs/Kiba/Extend/Transforms/MsAccess/ScientificNotationToNumberString.html
@@ -193,7 +193,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Prepend.html b/docs/Kiba/Extend/Transforms/Prepend.html
index 58f2fbc85..5dd63c1d9 100644
--- a/docs/Kiba/Extend/Transforms/Prepend.html
+++ b/docs/Kiba/Extend/Transforms/Prepend.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue.html b/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue.html
index 34e3cf9b3..f4f137bf4 100644
--- a/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue.html
+++ b/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue.html
@@ -432,7 +432,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue/MissingDelimiterError.html b/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue/MissingDelimiterError.html
index 505092f60..47bbda403 100644
--- a/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue/MissingDelimiterError.html
+++ b/docs/Kiba/Extend/Transforms/Prepend/FieldToFieldValue/MissingDelimiterError.html
@@ -223,7 +223,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Prepend/ToFieldValue.html b/docs/Kiba/Extend/Transforms/Prepend/ToFieldValue.html
index f833de2d7..42f1f02dc 100644
--- a/docs/Kiba/Extend/Transforms/Prepend/ToFieldValue.html
+++ b/docs/Kiba/Extend/Transforms/Prepend/ToFieldValue.html
@@ -280,7 +280,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Rename.html b/docs/Kiba/Extend/Transforms/Rename.html
index 58da1212d..410df24c8 100644
--- a/docs/Kiba/Extend/Transforms/Rename.html
+++ b/docs/Kiba/Extend/Transforms/Rename.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Rename/Field.html b/docs/Kiba/Extend/Transforms/Rename/Field.html
index 944713b35..0670e781b 100644
--- a/docs/Kiba/Extend/Transforms/Rename/Field.html
+++ b/docs/Kiba/Extend/Transforms/Rename/Field.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Replace.html b/docs/Kiba/Extend/Transforms/Replace.html
index 63fc3b545..5de7e2d6c 100644
--- a/docs/Kiba/Extend/Transforms/Replace.html
+++ b/docs/Kiba/Extend/Transforms/Replace.html
@@ -118,7 +118,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Replace/EmptyFieldValues.html b/docs/Kiba/Extend/Transforms/Replace/EmptyFieldValues.html
index 181500079..2adee1bad 100644
--- a/docs/Kiba/Extend/Transforms/Replace/EmptyFieldValues.html
+++ b/docs/Kiba/Extend/Transforms/Replace/EmptyFieldValues.html
@@ -195,7 +195,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Replace/FieldValueWithStaticMapping.html b/docs/Kiba/Extend/Transforms/Replace/FieldValueWithStaticMapping.html
index 7ef37ce9b..e4ced09de 100644
--- a/docs/Kiba/Extend/Transforms/Replace/FieldValueWithStaticMapping.html
+++ b/docs/Kiba/Extend/Transforms/Replace/FieldValueWithStaticMapping.html
@@ -209,7 +209,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Reshape.html b/docs/Kiba/Extend/Transforms/Reshape.html
index 8f21b12d1..86aca2ba0 100644
--- a/docs/Kiba/Extend/Transforms/Reshape.html
+++ b/docs/Kiba/Extend/Transforms/Reshape.html
@@ -118,7 +118,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Reshape/CollapseMultipleFieldsToOneTypedFieldPair.html b/docs/Kiba/Extend/Transforms/Reshape/CollapseMultipleFieldsToOneTypedFieldPair.html
index e7f7629d3..113a41605 100644
--- a/docs/Kiba/Extend/Transforms/Reshape/CollapseMultipleFieldsToOneTypedFieldPair.html
+++ b/docs/Kiba/Extend/Transforms/Reshape/CollapseMultipleFieldsToOneTypedFieldPair.html
@@ -377,7 +377,7 @@
diff --git a/docs/Kiba/Extend/Transforms/Split.html b/docs/Kiba/Extend/Transforms/Split.html
index cb8de6a94..e2da4eee6 100644
--- a/docs/Kiba/Extend/Transforms/Split.html
+++ b/docs/Kiba/Extend/Transforms/Split.html
@@ -116,7 +116,7 @@ Overview
diff --git a/docs/Kiba/Extend/Transforms/Split/IntoMultipleColumns.html b/docs/Kiba/Extend/Transforms/Split/IntoMultipleColumns.html
index 926918a64..4099b4da8 100644
--- a/docs/Kiba/Extend/Transforms/Split/IntoMultipleColumns.html
+++ b/docs/Kiba/Extend/Transforms/Split/IntoMultipleColumns.html
@@ -378,7 +378,7 @@
diff --git a/docs/Kiba/Extend/Utils.html b/docs/Kiba/Extend/Utils.html
index 5d710aa73..b95b3a24e 100644
--- a/docs/Kiba/Extend/Utils.html
+++ b/docs/Kiba/Extend/Utils.html
@@ -105,7 +105,7 @@ Defined Under Namespace
diff --git a/docs/Kiba/Extend/Utils/Lookup.html b/docs/Kiba/Extend/Utils/Lookup.html
index 26d1f25db..bcb425316 100644
--- a/docs/Kiba/Extend/Utils/Lookup.html
+++ b/docs/Kiba/Extend/Utils/Lookup.html
@@ -270,7 +270,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/CriteriaChecker.html b/docs/Kiba/Extend/Utils/Lookup/CriteriaChecker.html
index 60e7f664c..0919a4a1d 100644
--- a/docs/Kiba/Extend/Utils/Lookup/CriteriaChecker.html
+++ b/docs/Kiba/Extend/Utils/Lookup/CriteriaChecker.html
@@ -391,7 +391,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/FieldEmptiness.html b/docs/Kiba/Extend/Utils/Lookup/FieldEmptiness.html
index 2f15960ae..c8d716b86 100644
--- a/docs/Kiba/Extend/Utils/Lookup/FieldEmptiness.html
+++ b/docs/Kiba/Extend/Utils/Lookup/FieldEmptiness.html
@@ -285,7 +285,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/MultivalPairs.html b/docs/Kiba/Extend/Utils/Lookup/MultivalPairs.html
index 32a1f79d4..76bf4f2df 100644
--- a/docs/Kiba/Extend/Utils/Lookup/MultivalPairs.html
+++ b/docs/Kiba/Extend/Utils/Lookup/MultivalPairs.html
@@ -321,7 +321,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/PairEquality.html b/docs/Kiba/Extend/Utils/Lookup/PairEquality.html
index a6eb072dd..22605ae45 100644
--- a/docs/Kiba/Extend/Utils/Lookup/PairEquality.html
+++ b/docs/Kiba/Extend/Utils/Lookup/PairEquality.html
@@ -335,7 +335,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/PairInclusion.html b/docs/Kiba/Extend/Utils/Lookup/PairInclusion.html
index 65d680a03..937b017b9 100644
--- a/docs/Kiba/Extend/Utils/Lookup/PairInclusion.html
+++ b/docs/Kiba/Extend/Utils/Lookup/PairInclusion.html
@@ -333,7 +333,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/RowSelector.html b/docs/Kiba/Extend/Utils/Lookup/RowSelector.html
index ac2876036..67e1cedfd 100644
--- a/docs/Kiba/Extend/Utils/Lookup/RowSelector.html
+++ b/docs/Kiba/Extend/Utils/Lookup/RowSelector.html
@@ -274,7 +274,7 @@
diff --git a/docs/Kiba/Extend/Utils/Lookup/SetChecker.html b/docs/Kiba/Extend/Utils/Lookup/SetChecker.html
index 138135601..23888fb94 100644
--- a/docs/Kiba/Extend/Utils/Lookup/SetChecker.html
+++ b/docs/Kiba/Extend/Utils/Lookup/SetChecker.html
@@ -441,7 +441,7 @@
diff --git a/docs/_index.html b/docs/_index.html
index 4902cbd52..d24c174cd 100644
--- a/docs/_index.html
+++ b/docs/_index.html
@@ -403,16 +403,16 @@ Namespace Listing A-Z
- Fields Fields (Kiba::Extend::Transforms::Delete)
+ (Kiba::Extend::Transforms::Deduplicate)
- Fields Fields (Kiba::Extend::Transforms::Deduplicate)
+ (Kiba::Extend::Transforms::Delete)
@@ -489,9 +489,9 @@ Namespace Listing A-Z
- Helpers Helpers (Kiba::Extend::Transforms::Clean)
+ (Kiba::Extend::Transforms)
@@ -740,16 +740,16 @@ Namespace Listing A-Z
- ToFieldValue ToFieldValue (Kiba::Extend::Transforms::Append)
+ (Kiba::Extend::Transforms::Prepend)
- ToFieldValue ToFieldValue (Kiba::Extend::Transforms::Prepend)
+ (Kiba::Extend::Transforms::Append)
@@ -787,7 +787,7 @@ Namespace Listing A-Z
diff --git a/docs/class_list.html b/docs/class_list.html
index 020572104..0babd6866 100644
--- a/docs/class_list.html
+++ b/docs/class_list.html
@@ -43,7 +43,7 @@
CSV < Object
Kiba::Extend::Destinations Append Kiba::Extend::Transforms NilFields < Object
Kiba::Extend::Transforms::Append Clean Kiba::Extend::Transforms Helpers Kiba::Extend::Transforms::Clean FullRecord < Object
Kiba::Extend::Transforms::CombineValues Copy Kiba::Extend::Transforms Field < Object
Kiba::Extend::Transforms::Copy Cspace Kiba::Extend::Transforms FieldValues < Object
Kiba::Extend::Transforms::Deduplicate Fields < Object
Kiba::Extend::Transforms::Deduplicate Flag < Object
Kiba::Extend::Transforms::Deduplicate Delete Kiba::Extend::Transforms Fields < Object
Kiba::Extend::Transforms::Delete Merge Kiba::Extend::Transforms Rename Kiba::Extend::Transforms Field < Object
Kiba::Extend::Transforms::Rename Split Kiba::Extend::Transforms CSV < Object
Kiba::Extend::Destinations Append Kiba::Extend::Transforms NilFields < Object
Kiba::Extend::Transforms::Append Clean Kiba::Extend::Transforms FullRecord < Object
Kiba::Extend::Transforms::CombineValues Copy Kiba::Extend::Transforms Field < Object
Kiba::Extend::Transforms::Copy Cspace Kiba::Extend::Transforms FieldValues < Object
Kiba::Extend::Transforms::Deduplicate Fields < Object
Kiba::Extend::Transforms::Deduplicate Flag < Object
Kiba::Extend::Transforms::Deduplicate Delete Kiba::Extend::Transforms Fields < Object
Kiba::Extend::Transforms::Delete Merge Kiba::Extend::Transforms Rename Kiba::Extend::Transforms Field < Object
Kiba::Extend::Transforms::Rename Split Kiba::Extend::Transforms
diff --git a/docs/file.LICENSE.html b/docs/file.LICENSE.html
index 65a4fdf99..9edbed982 100644
--- a/docs/file.LICENSE.html
+++ b/docs/file.LICENSE.html
@@ -60,7 +60,7 @@
The MIT License (MIT)
Kiba::Extend
diff --git a/docs/index.html b/docs/index.html
index cda75b02f..52b58654c 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -76,7 +76,7 @@ Kiba::Extend
diff --git a/docs/method_list.html b/docs/method_list.html
index 7ef008816..4f435c93a 100644
--- a/docs/method_list.html
+++ b/docs/method_list.html
@@ -94,13 +94,21 @@
+
+
+
+
+
#fields Kiba::Extend::Fieldset
@@ -108,7 +116,7 @@
-
+
#filename Kiba::Extend::Destinations::CSV
@@ -116,7 +124,7 @@
-
+
#hash Kiba::Extend::Fieldset
@@ -124,7 +132,7 @@
-
+
#headers Kiba::Extend::Destinations::CSV
@@ -132,7 +140,7 @@
-
+
#initialize Kiba::Extend::Transforms::CombineValues::FullRecord
@@ -140,7 +148,7 @@
-
+
#initialize Kiba::Extend::Utils::Lookup::SetChecker
@@ -148,7 +156,7 @@
-
+
#initialize Kiba::Extend::Utils::Lookup::MultivalPairs
@@ -156,6 +164,14 @@
+
+
+
#initialize
+
Kiba::Extend::Utils::Lookup::FieldEmptiness
+
+
+
#initialize
@@ -277,14 +293,6 @@
-
-
#initialize
-
Kiba::Extend::Utils::Lookup::FieldEmptiness
-
-
-
-
#initialize Kiba::Extend::Utils::Lookup::PairEquality
@@ -292,7 +300,7 @@
-
+
#initialize Kiba::Extend::Utils::Lookup::PairInclusion
@@ -300,7 +308,7 @@
-
+
#initialize Kiba::Extend::Utils::Lookup::CriteriaChecker
@@ -308,7 +316,7 @@
-
+
#initialize Kiba::Extend::Utils::Lookup::RowSelector
@@ -316,7 +324,7 @@
-
+
#initialize Kiba::Extend::Transforms::Copy::Field
@@ -324,7 +332,7 @@
-
+
#initialize Kiba::Extend::Destinations::CSV
@@ -332,7 +340,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::AlphabetizeFieldValues
@@ -340,7 +348,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::ClearFields
@@ -348,7 +356,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::DelimiterOnlyFields
@@ -356,7 +364,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::DowncaseFieldValues
@@ -364,7 +372,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::EmptyFieldGroups
@@ -372,7 +380,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::RegexpFindReplaceFieldVals
@@ -380,7 +388,7 @@
-
+
#initialize Kiba::Extend::Transforms::Clean::StripFields
@@ -388,7 +396,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::CompareFieldsFlag
@@ -396,7 +404,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::ConstantValue
@@ -404,7 +412,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::ConstantValueConditional
@@ -412,7 +420,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::CountOfMatchingRows
@@ -420,7 +428,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::MultivalueConstant
@@ -428,7 +436,7 @@
-
+
#initialize Kiba::Extend::Transforms::Merge::MultiRowLookup
@@ -436,7 +444,7 @@
-
+
#initialize Kiba::Extend::Transforms::Split::IntoMultipleColumns
@@ -444,7 +452,7 @@
-
+
#initialize Kiba::Extend::Transforms::Append::NilFields
@@ -452,7 +460,7 @@
-
+
#initialize Kiba::Extend::Transforms::Append::ToFieldValue
@@ -460,7 +468,7 @@
-
+
#initialize Kiba::Extend::Transforms::Cspace::ConvertToID
@@ -468,7 +476,7 @@
-
+
#initialize Kiba::Extend::Transforms::Cspace::FlagInvalidCharacters
@@ -476,7 +484,7 @@
-
+
#initialize Kiba::Extend::Transforms::Cspace::NormalizeForID
@@ -484,7 +492,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::EmptyFieldValues
@@ -492,7 +500,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::Fields
@@ -500,7 +508,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::FieldsExcept
@@ -508,7 +516,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::FieldValueContainingString
@@ -516,7 +524,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::FieldValueIfEqualsOtherField
@@ -524,7 +532,7 @@
-
+
#initialize Kiba::Extend::Transforms::Delete::FieldValueMatchingRegexp
@@ -532,7 +540,7 @@
-
+
#initialize Kiba::Extend::Transforms::Rename::Field
@@ -540,7 +548,7 @@
-
+
#initialize Kiba::Extend::Transforms::Explode::RowsFromMultivalField
@@ -548,7 +556,7 @@
-
+
#initialize Kiba::Extend::Transforms::Explode::ColumnsRemappedInNewRows
@@ -556,7 +564,7 @@
-
+
#initialize Kiba::Extend::Transforms::Explode::FieldValuesToNewRows
@@ -564,7 +572,7 @@
-
+
#initialize Kiba::Extend::Transforms::Prepend::FieldToFieldValue::MissingDelimiterError
@@ -572,7 +580,7 @@
-
+
#initialize Kiba::Extend::Transforms::Prepend::FieldToFieldValue
@@ -580,7 +588,7 @@
-
+
#initialize Kiba::Extend::Transforms::Prepend::ToFieldValue
@@ -588,7 +596,7 @@
-
+
#join_values Kiba::Extend::Fieldset
@@ -596,7 +604,7 @@
-
+
#populate Kiba::Extend::Fieldset
@@ -604,7 +612,7 @@
-
+
#process Kiba::Extend::Transforms::Explode::RowsFromMultivalField
@@ -612,7 +620,7 @@
-
+
#process Kiba::Extend::Transforms::Explode::FieldValuesToNewRows
@@ -620,14 +628,6 @@
-
-
-
#result
-
Kiba::Extend::Utils::Lookup::SetChecker
-
-
-
#result
@@ -638,32 +638,32 @@
-
#result
-
Kiba::Extend::Utils::Lookup::MultivalPairs
+
#result
+
Kiba::Extend::Utils::Lookup::PairEquality
-
#result
-
Kiba::Extend::Utils::Lookup::RowSelector
+
#result
+
Kiba::Extend::Utils::Lookup::FieldEmptiness
-
#result
-
Kiba::Extend::Utils::Lookup::FieldEmptiness
+
#result
+
Kiba::Extend::Utils::Lookup::MultivalPairs
-
#result
-
Kiba::Extend::Utils::Lookup::PairEquality
+
#result
+
Kiba::Extend::Utils::Lookup::SetChecker
@@ -677,6 +677,14 @@
+
+
#result
+
Kiba::Extend::Utils::Lookup::RowSelector
+
+
+
+
#set_type Kiba::Extend::Utils::Lookup::SetChecker
@@ -684,7 +692,7 @@
-
+
#type Kiba::Extend::Utils::Lookup::CriteriaChecker
@@ -692,7 +700,7 @@
-
+
#value_ct Kiba::Extend::Fieldset
@@ -700,7 +708,7 @@
-
+
#write Kiba::Extend::Destinations::CSV
diff --git a/docs/top-level-namespace.html b/docs/top-level-namespace.html
index 745f8b5f2..c69e8d5ca 100644
--- a/docs/top-level-namespace.html
+++ b/docs/top-level-namespace.html
@@ -98,12 +98,34 @@
CSVOPT =
-
+
+
+
+
Default CSV options
+
+
+
+
+
+
+
{ headers: true , header_converters: :symbol } . freeze DELIM =
-
+
+
+
+
Default multivalue delimter for splitting and joining multiple values in a field
+
+
+
+
+
+
+
' ; '
diff --git a/lib/kiba/extend.rb b/lib/kiba/extend.rb
index df0157547..007cd62ea 100644
--- a/lib/kiba/extend.rb
+++ b/lib/kiba/extend.rb
@@ -8,7 +8,10 @@
require 'xxhash'
require 'facets/kernel/blank'
+# Default CSV options
CSVOPT = { headers: true, header_converters: :symbol }.freeze
+
+# Default multivalue delimter for splitting and joining multiple values in a field
DELIM = ';'
# The Kiba ETL framework for Ruby.
@@ -25,6 +28,7 @@ module Extend
require 'kiba/extend/fieldset'
require 'kiba/extend/destinations/csv'
+ require 'kiba/extend/transforms/helpers'
require 'kiba/extend/transforms/append'
require 'kiba/extend/transforms/clean'
require 'kiba/extend/transforms/combine_values'
diff --git a/lib/kiba/extend/transforms/clean.rb b/lib/kiba/extend/transforms/clean.rb
index 2d65eff73..481fa6d36 100644
--- a/lib/kiba/extend/transforms/clean.rb
+++ b/lib/kiba/extend/transforms/clean.rb
@@ -3,38 +3,49 @@
module Kiba
module Extend
module Transforms
+ # Transformations to clean up data
module Clean
::Clean = Kiba::Extend::Transforms::Clean
- module Helpers
- def delim_only?(val, delim, usenull = false)
- chk = val.gsub(delim, '').strip
- chk = chk.gsub('%NULLVALUE%', '').strip if usenull
- chk.empty? ? true : false
- end
- end
-
+ # @note This transformation does **NOT** sort the **ROWS** in a dataset. It sorts values within
+ # individual fields of a row
+ # Sorts the multiple values within a field alphabetically
+ # @param fields [Array(Symbol)] names of fields to sort
+ # @param delim [String] Character(s) on which to split field values
+ # @param usenull [Boolean] Whether to treat %NULLVALUE% as a blank in processing
+ # @param direction [:asc, :desc] Direction in which to sort field values
class AlphabetizeFieldValues
- include Clean::Helpers
- def initialize(fields:, delim:)
+ include Kiba::Extend::Transforms::Helpers
+ def initialize(fields:, delim:, usenull: false, direction: :asc)
@fields = fields
@delim = delim
+ @usenull = usenull
+ @direction = direction
end
# @private
def process(row)
- @fields.each do |field|
- vals = row.fetch(field, nil)
- next if vals.blank?
- next if delim_only?(vals, @delim)
+ field_values(row: row, fields: @fields, delim: @delim, usenull: @usenull).each do |field, val|
+ next unless val[@delim]
- vals = vals.split(@delim)
- next if vals.size == 1
-
- row[field] = vals.sort_by(&:downcase).join(@delim)
+ row[field] = sort_values(val.split(@delim)).join(@delim)
end
row
end
+
+ private
+
+ def process_for_sort(val)
+ return val.gsub('%NULLVALUE%', 'zzzzzzzzzzzzzz').downcase.gsub(/[^[:alnum:][:space:]]/, '') if @usenull
+
+ val.downcase.gsub(/[^[:alnum:][:space:]]/, '')
+ end
+
+ def sort_values(vals)
+ return vals.sort_by { |v| process_for_sort(v) } if @direction == :asc
+
+ vals.sort_by { |v| process_for_sort(v) }.reverse
+ end
end
class ClearFields
@@ -57,7 +68,7 @@ def process(row)
end
class DelimiterOnlyFields
- include Clean::Helpers
+ include Kiba::Extend::Transforms::Helpers
def initialize(delim:, use_nullvalue: false)
@delim = delim
@use_nullvalue = use_nullvalue
diff --git a/lib/kiba/extend/transforms/helpers.rb b/lib/kiba/extend/transforms/helpers.rb
new file mode 100644
index 000000000..f7afa13fd
--- /dev/null
+++ b/lib/kiba/extend/transforms/helpers.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Kiba
+ module Extend
+ module Transforms
+ # utility functions across Transforms
+ module Helpers
+ extend self
+ # Indicates whether a field value is delimiter-only. If `usenull` is set to true, the
+ # %NULLVALUE% string is treated as empty in detecting delimiter-only-ness
+ # @param val [String] The field value to check
+ # @param delim [String] The multivalue delimiter
+ # @param usenull [Boolean] If true, replaces '%NULLVALUE%' with '' to make determination
+ # @return [false] if `value` is nil, empty, or contains characters other than delimiter(s)
+ # and leading/trailing spaces
+ # @return [true] if `value` contains only delimiter(s) and leading/trailing spaces
+ def delim_only?(val, delim, usenull = false)
+ return false if val.nil?
+ return false if val.strip.empty?
+
+ chk = val.gsub(delim, '').strip
+ chk = chk.gsub('%NULLVALUE%', '').strip if usenull
+ chk.empty?
+ end
+
+ # @param row [Hash{Symbol=>String,Nil}l] A row of data
+ # @param fields [Array(Symbol)] Names of fields to process
+ # @param discard [:nil, :empty, :delim] Types of field values to remove from returned hash
+ # @param delim [String] Multivalue delimiter used to split fields
+ # @param usenull [Boolean] If true, replaces '%NULLVALUE%' with '' to make determination
+ # @return [Hash{Symbol=>String,Nil}l] of field data for fields that meet keep criteria
+ def field_values(row:, fields:, discard: %i[nil empty delim], delim: DELIM, usenull: false)
+ field_vals = fields.map{ |field| [field, row.fetch(field, nil)] }.to_h
+ return field_vals if discard.blank?
+
+ field_vals = field_vals.reject{ |field, val| val.nil? } if discard.any?(:nil)
+ keep = keep_fields(field_vals, discard, delim, usenull)
+ field_vals.select{ |field, val| keep.any?(field) }
+ end
+
+ # @param field_vals [Hash{Symbol=>String,Nil}l] A subset of a row
+ # @param discard [:nil, :empty, :delim] Types of field values to remove from returned hash
+ # @param delim [String] Multivalue delimiter used to split fields
+ # @param usenull [Boolean] If true, replaces '%NULLVALUE%' with '' to make determination
+ # @return [Array(Symbol)] of names of fields that should be kept, based on given discard
+ # and usenull param values and the field values
+ private_class_method def keep_fields(field_vals, discard, delim, usenull)
+ field_vals = field_vals.transform_values{ |val| val.gsub('%NULLVALUE%', '') } if usenull
+ field_vals = field_vals.reject{ |field, val| val.empty? } if discard.any?(:empty)
+ field_vals = field_vals.reject{ |field, val| delim_only?(val, delim) } if discard.any?(:delim)
+ field_vals.keys
+ end
+ end
+ end
+ end
+end
diff --git a/lib/kiba/extend/version.rb b/lib/kiba/extend/version.rb
index 36b903c67..e8b58c5f3 100644
--- a/lib/kiba/extend/version.rb
+++ b/lib/kiba/extend/version.rb
@@ -2,6 +2,6 @@
module Kiba
module Extend
- VERSION = '1.15.2'
+ VERSION = '1.15.3'
end
end
diff --git a/spec/kiba/extend/transforms/clean_spec.rb b/spec/kiba/extend/transforms/clean_spec.rb
index 9973fecd0..0f5abe56c 100644
--- a/spec/kiba/extend/transforms/clean_spec.rb
+++ b/spec/kiba/extend/transforms/clean_spec.rb
@@ -6,32 +6,85 @@
describe 'AlphabetizeFieldValues' do
test_csv = 'tmp/test.csv'
rows = [
- %w[id type],
- ['1', 'Person;unmapped;Organization'],
- ['2', ';'],
- ['3', nil],
- ['4', ''],
- ['5', 'Person;notmapped']
+ %w[type],
+ ['Person;unmapped;Organization'],
+ [';'],
+ [nil],
+ [''],
+ ['Person;notmapped'],
+ ['%NULLVALUE%;apple'],
+ ['oatmeal;%NULLVALUE%']
]
before { generate_csv(test_csv, rows) }
+ let(:usenull) { false }
+ let(:direction) { :asc }
let(:result) do
execute_job(filename: test_csv,
xform: Clean::AlphabetizeFieldValues,
- xformopt: { fields: %i[type], delim: ';' })
+ xformopt: { fields: %i[type], delim: ';', usenull: usenull, direction: direction })
end
- it 'sorts field values alphabetically' do
- expect(result[0][:type]).to eq('Organization;Person;unmapped')
- expect(result[4][:type]).to eq('notmapped;Person')
- end
- it 'leaves delimiter-only fields alone' do
- expect(result[1][:type]).to eq(';')
- end
- it 'leaves nil fields alone' do
- expect(result[2][:type]).to be_nil
+ context 'when usenull = false' do
+ it 'sorts as expected' do
+ expected = [
+ 'Organization;Person;unmapped',
+ ';',
+ nil,
+ '',
+ 'notmapped;Person',
+ 'apple;%NULLVALUE%',
+ '%NULLVALUE%;oatmeal'
+ ]
+ expect(result.map{ |res| res[:type] }).to eq(expected)
+ end
+
+ context 'when direction = :desc' do
+ let(:direction) { :desc }
+ it 'sorts as expected' do
+ expected = [
+ 'unmapped;Person;Organization',
+ ';',
+ nil,
+ '',
+ 'Person;notmapped',
+ '%NULLVALUE%;apple',
+ 'oatmeal;%NULLVALUE%'
+ ]
+ expect(result.map{ |res| res[:type] }).to eq(expected)
+ end
+ end
end
- it 'leaves empty string fields alone' do
- expect(result[3][:type]).to eq('')
+
+ context 'when usenull = true' do
+ let(:usenull) { true }
+ it 'sorts as expected' do
+ expected = [
+ 'Organization;Person;unmapped',
+ ';',
+ nil,
+ '',
+ 'notmapped;Person',
+ 'apple;%NULLVALUE%',
+ 'oatmeal;%NULLVALUE%'
+ ]
+ expect(result.map{ |res| res[:type] }).to eq(expected)
+ end
+
+ context 'when direction = :desc' do
+ let(:direction) { :desc }
+ it 'sorts as expected' do
+ expected = [
+ 'unmapped;Person;Organization',
+ ';',
+ nil,
+ '',
+ 'Person;notmapped',
+ '%NULLVALUE%;apple',
+ '%NULLVALUE%;oatmeal'
+ ]
+ expect(result.map{ |res| res[:type] }).to eq(expected)
+ end
+ end
end
end
@@ -60,13 +113,6 @@
result = execute_job(filename: test_csv,
xform: Clean::ClearFields,
xformopt: { fields: %i[type], if_equals: ';' })
- expected = [
- { id: '1', type: 'Person;unmapped;Organization' },
- { id: '2', type: nil },
- { id: '3', type: nil },
- { id: '4', type: '' },
- { id: '5', type: 'Person;notmapped' }
- ]
expect(result.map { |e| e[1] }.uniq[0]).to be_nil
end
end
diff --git a/spec/kiba/extend/transforms/helpers_spec.rb b/spec/kiba/extend/transforms/helpers_spec.rb
new file mode 100644
index 000000000..6cbe4db72
--- /dev/null
+++ b/spec/kiba/extend/transforms/helpers_spec.rb
@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Kiba::Extend::Transforms::Helpers do
+ include Kiba::Extend::Transforms::Helpers
+
+ describe '#delim_only?' do
+ context 'when delim = |' do
+ let(:delim) { '|' }
+ let(:usenull) { false }
+ let(:result) { delim_only?(value, delim, usenull) }
+ context 'when value = foo|bar' do
+ let(:value) { 'foo|bar' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = |' do
+ let(:value) { '|' }
+ it 'returns true' do
+ expect(result).to be true
+ end
+ end
+ context 'when value = %NULLVALUE%|%NULLVALUE%' do
+ let(:value) { '%NULLVALUE%|%NULLVALUE%' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = %NULLVALUE%' do
+ let(:value) { '%NULLVALUE%' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = %NULLVALUE%|blah' do
+ let(:value) { '%NULLVALUE%|blah' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = blank' do
+ let(:value) { '' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = nil' do
+ let(:value) { nil }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ context 'when value = " "' do
+ let(:value) { ' ' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+
+ context 'when usenull = true' do
+ let(:usenull) { true }
+ context 'when value = %NULLVALUE%|%NULLVALUE%' do
+ let(:value) { '%NULLVALUE%|%NULLVALUE%' }
+ it 'returns true' do
+ expect(result).to be true
+ end
+ end
+ context 'when value = %NULLVALUE%' do
+ let(:value) { '%NULLVALUE%' }
+ it 'returns true' do
+ expect(result).to be true
+ end
+ end
+ context 'when value = %NULLVALUE%|blah' do
+ let(:value) { '%NULLVALUE%|blah' }
+ it 'returns false' do
+ expect(result).to be false
+ end
+ end
+ end
+ end
+ end
+
+ describe '#field_values' do
+ let(:row) { {
+ a: nil,
+ b: '',
+ c: ';',
+ d: 'foo',
+ e: '%NULLVALUE%',
+ f: '%NULLVALUE%;%NULLVALUE%'
+ } }
+ let(:fields) { %i[a b c d e f] }
+ let(:discard) { %i[nil empty delim] }
+ let(:delim) { ';' }
+ let(:usenull) { false }
+ let(:result) { field_values(row: row, fields: fields, discard: discard, delim: delim, usenull: usenull) }
+ context 'with all fields and no discard values' do
+ let(:discard) { %i[] }
+ it 'returns all field values' do
+ expect(result).to eq(row)
+ end
+ end
+
+ context 'with all fields and default discard and usenull values' do
+ it 'returns expected field values' do
+ expected = {
+ d: 'foo',
+ e: '%NULLVALUE%',
+ f: '%NULLVALUE%;%NULLVALUE%'
+ }
+ expect(result).to eq(expected)
+ end
+ end
+
+ context 'with all fields, default discard, and usenull true' do
+ let(:usenull) { true }
+ it 'returns expected field values' do
+ expected = {
+ d: 'foo'
+ }
+ expect(result).to eq(expected)
+ end
+ end
+
+ context 'with non-existent field, default discard, and usenull true' do
+ let(:usenull) { true }
+ let(:fields) { %i[d q] }
+ it 'returns expected field values' do
+ expected = {
+ d: 'foo'
+ }
+ expect(result).to eq(expected)
+ end
+ end
+ end
+end