Merge pull request #40 from lyrasis/json-dest-job-int

JsonArray destination added; Ruby 3.0 compatible arguments

.ruby-version

@@ -0,0 +1 @@
+3.0.2

Gemfile

@@ -9,7 +9,7 @@ group :development, :test do
   gem 'byebug', '~>11.0'
   gem 'pry', '~> 0.14'
   gem 'rake', '~> 13.0'
-  gem 'rspec', '~> 3.0'
+  gem 'rspec', '~> 3.10'
   gem 'rubocop', '~> 1.18.4'
   gem 'rubocop-rspec', '~> 2.4.0'
 end

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kiba-extend (2.4.1)
+    kiba-extend (2.5.0)
       activesupport (~> 6.1.4)
       csv (~> 3.0)
       dry-configurable (~> 0.11)
@@ -25,7 +25,7 @@ GEM
     coderay (1.1.3)
     concurrent-ruby (1.1.9)
     csv (3.2.0)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
     docile (1.4.0)
     dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
@@ -52,19 +52,19 @@ GEM
     rake (13.0.1)
     regexp_parser (2.1.1)
     rexml (3.2.5)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
     rubocop (1.18.4)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -102,7 +102,7 @@ DEPENDENCIES
   kiba-extend!
   pry (~> 0.14)
   rake (~> 13.0)
-  rspec (~> 3.0)
+  rspec (~> 3.10)
   rubocop (~> 1.18.4)
   rubocop-rspec (~> 2.4.0)
   simplecov

kiba-extend.gemspec

@@ -43,12 +43,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'kiba-common', '~> 1.5.0'
   spec.add_dependency 'thor', '~> 1.1.0'
   spec.add_dependency 'xxhash', '~> 0.4'
-
-  # spec.add_development_dependency 'bundler', '>= 1.17'
-  # spec.add_development_dependency 'byebug', '~>11.0'
-  # spec.add_development_dependency 'pry', '~> 0.14'
-  # spec.add_development_dependency 'rake', '~> 13.0'
-  # spec.add_development_dependency 'rspec', '~> 3.0'
-  # spec.add_development_dependency 'rubocop', '~> 1.18.4'
-  # spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0'
 end

lib/kiba/extend.rb

@@ -14,6 +14,9 @@
 
 require 'kiba/extend/registry/file_registry'
 
+# These are still here to support legacy projects/unconverted tests.
+# Do not call these constants in new code.
+# Use Kiba::Extend.csvopts and Kiba::Extend.delim instead
 # Default CSV options
 CSVOPT = { headers: true, header_converters: :symbol }.freeze
 

lib/kiba/extend/destinations/csv.rb

@@ -25,7 +25,7 @@ def initialize(filename:, csv_options: nil, headers: nil, initial_headers: [])
 
         # @private
         def write(row)
-          @csv ||= ::CSV.open(filename, 'wb', csv_options)
+          @csv ||= ::CSV.open(filename, 'wb', **csv_options)
           @headers ||= row.keys
           order_headers
           @headers_written ||= (csv << headers; true)

lib/kiba/extend/destinations/json_array.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Kiba
+  module Extend
+    module Destinations
+      # Writes each row as a valid JSON object that is an element in a JSON array
+      #
+      # This is simliar to the idea of, but not technically compliant with, [JSON Lines](https://jsonlines.org/)
+      class JsonArray
+        # @param filename [String] path for writing JSON file
+        def initialize(filename:)
+          @json = []
+          @file = open(filename, 'w')
+        end
+
+        # @private
+        def write(row)
+          @json << row
+        end
+
+        # @private
+        def close
+          @file << @json.to_json
+          @file.close
+        end
+      end
+    end
+  end
+end

lib/kiba/extend/jobs/runner.rb

@@ -33,22 +33,32 @@ def add_lookup(config)
             if instance_variable_defined?(key_as_iv)
               instance_variable_get(key_as_iv)
             else
-              instance_variable_set(key_as_iv, Lookup.csv_to_hash(config.args))
+              instance_variable_set(key_as_iv, Lookup.csv_to_hash(**config.args))
             end
           }
         end
 
         # This stuff does not get handled by parsing source code
-        def add_source_and_destination
-          %i[config sources destinations].compact.each do |method_name|
-            populate_control(method(method_name))
-          end
+        def add_config
+          return if config.empty?
+
+          control.config.merge(config)
+        end
+
+        def add_destinations
+          context.instance_eval(destinations)
+        end
+
+        def add_sources
+          context.instance_eval(sources)
         end
 
         def assemble_control
           lookups_to_memoized_methods if @files[:lookup]
           parse_job_segments
-          add_source_and_destination
+          add_config
+          add_sources
+          add_destinations
           add_decoration
           control
         end
@@ -64,6 +74,8 @@ def check_requirements
 
         def destinations
           @files[:destination].map { |config| file_config(config) }
+            .map{ |src| "destination #{src[:klass]}, **#{src[:args]}" }
+            .join("\n")
         end
 
         def file_config(config)
@@ -87,20 +99,6 @@ def parse_job_segments
           parse_job(control, context, [pre_process, transform, post_process])
         end
 
-        # @todo I think this may not need to be so opaque and metamagicky now that this module has
-        #   direct access to one instance of context and control for all the processes, but it works
-        #   now. Test/simplify later.
-        def populate_control(method)
-          elements = method.call
-          control_method = control.method(method.name)
-          target = control_method.call
-          if method.name == :config
-            target.merge(elements)
-          else
-            elements.each { |element| target << element }
-          end
-        end
-
         def show_me_decoration
           return unless Kiba::Extend.job.show_me
 
@@ -110,6 +108,8 @@ def show_me_decoration
 
         def sources
           @files[:source].map { |config| file_config(config) }
+            .map{ |src| "source #{src[:klass]}, **#{src[:args]}" }
+            .join("\n")
         end
 
         def tell_me_decoration

lib/kiba/extend/registry/file_registry_entry.rb

@@ -108,12 +108,12 @@ def set_defaults
           @creator = nil
           @desc = ''
           @dest_class = Kiba::Extend.destination
-          @dest_opt = Kiba::Extend.csvopts
+          @dest_opt = nil
           @dest_special_opts = nil
           @lookup_on = nil
           @path = nil
           @src_class = Kiba::Extend.source
-          @src_opt = Kiba::Extend.csvopts
+          @src_opt = nil
           @supplied = false
           @tags = []
           @valid = false

lib/kiba/extend/registry/registered_destination.rb

@@ -9,13 +9,13 @@ module Registry
       class RegisteredDestination < RegisteredFile
         # Arguments for calling Kiba Destination class
         def args
-          return [simple_args] unless @data.dest_special_opts
+          return simple_args unless @data.dest_special_opts
 
           opts = supported_special_opts
           warn_about_opts if opts.length < @data.dest_special_opts.length
-          return [simple_args] if opts.empty?
+          return simple_args if opts.empty?
 
-          [simple_args.merge(supported_special_opts)]
+          simple_args.merge(supported_special_opts)
         end
 
         # Description of file
@@ -46,7 +46,7 @@ def klass_opts
         def simple_args
           return { filename: path }.merge(options_label(klass) => @data.dest_opt) if @data.dest_opt
 
-          { filename: path }.merge(labeled_options(klass))
+          default_args(klass)
         end
 
         def supported_special_opts

lib/kiba/extend/registry/registered_lookup.rb

@@ -41,8 +41,17 @@ def initialize(key:, data:)
         # Arguments for calling {Kiba::Extend::Lookup} with this file
         # @return [Hash]
         def args
-          opts = @data.src_opt ? @data.src_opt : file_options(@data.src_class)
-          { file: path, csvopt: opts, keycolumn: @data.lookup_on }
+          { file: path, keycolumn: @data.lookup_on }.merge(options)
+        end
+
+        private
+
+        def options
+          klass = @data.src_class
+          label = lookup_options_label(klass)
+          return {label=>@data.src_opt} if @data.src_opt
+
+          {label=>default_file_options(klass)}
         end
       end
     end

lib/kiba/extend/registry/registered_source.rb

@@ -15,7 +15,7 @@ class RegisteredSource < RegisteredFile
         # @return [Hash]
         def args
           opts = @data.src_opt ? { options_label(klass) => @data.src_opt } : labeled_options(klass)
-          [{ filename: path }.merge(opts)]
+          { filename: path }.merge(opts)
         end
 
         # Kiba Source class to call

lib/kiba/extend/registry/registry_entry_selector.rb

@@ -18,7 +18,9 @@ def created_by_class(cstr)
         # @return [Array<FileRegistryEntry>]
         def created_by_method(mstr)
           matcher = "#<Method: #{mstr}("
-          with_creator.select { |entry| entry.creator.to_s[matcher] }
+          with_creator.select { |entry|
+          binding.pry if mstr.end_with?('.csv_to_hash')
+            entry.creator.to_s[matcher] }
         end
 
         # Selects entries whose tags include all given tags