diff --git a/Gemfile.lock b/Gemfile.lock index d88e9fe..4dc585e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - parqueteur (1.3.2) + parqueteur (1.3.3) red-parquet (~> 6.0) GEM diff --git a/examples/hello-world.rb b/examples/hello-world.rb index a7456d8..18b9788 100644 --- a/examples/hello-world.rb +++ b/examples/hello-world.rb @@ -1,3 +1,4 @@ +require 'securerandom' require 'bundler/setup' require 'parqueteur' @@ -17,14 +18,16 @@ class FooParquetConverter < Parqueteur::Converter column :my_timestamp, :timestamp end -data = 100.times.collect do |i| +data = 1000.times.collect do |i| { 'id' => i, 'my_string_array' => %w[a b c], 'my_date' => Date.today, 'my_decimal' => BigDecimal('0.03'), 'my_int' => rand(1..10), - 'my_map' => { 'a' => 'b' }, + 'my_map' => 20.times.each_with_object({}) do |idx, hash| + hash["k_#{idx}"] = SecureRandom.urlsafe_base64(64) + end, 'my_string' => 'Hello World', 'my_struct' => { 'my_struct_str' => 'Hello World', diff --git a/lib/parqueteur/converter.rb b/lib/parqueteur/converter.rb index a5573fa..2476dc5 100644 --- a/lib/parqueteur/converter.rb +++ b/lib/parqueteur/converter.rb @@ -2,7 +2,7 @@ module Parqueteur class Converter - DEFAULT_BATCH_SIZE = 10 + DEFAULT_BATCH_SIZE = 100 def self.inline(&block) Class.new(self, &block) diff --git a/lib/parqueteur/types/map_type.rb b/lib/parqueteur/types/map_type.rb index 86f896c..8ff7a0a 100644 --- a/lib/parqueteur/types/map_type.rb +++ b/lib/parqueteur/types/map_type.rb @@ -4,18 +4,7 @@ module Parqueteur module Types class MapType < Parqueteur::Type def build_value_array(values) - builder = Arrow::MapArrayBuilder.new(arrow_type) - values.each do |entry| - builder.append_value - next if entry.nil? - - entry.each do |k, v| - builder.key_builder.append(k) - builder.item_builder.append(v) - end - end - - builder.finish + Arrow::MapArrayBuilder.build(arrow_type, values) end def arrow_type_builder diff --git a/lib/parqueteur/version.rb b/lib/parqueteur/version.rb index e299397..f2faaac 100644 --- a/lib/parqueteur/version.rb +++ b/lib/parqueteur/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Parqueteur - VERSION = '1.3.2' + VERSION = '1.3.3' end