From 1edbaa50e1eeccc5f4959082115fa8c3d0927daa Mon Sep 17 00:00:00 2001 From: nixx Date: Wed, 23 Oct 2024 14:00:11 +0300 Subject: [PATCH] Support codec column #135, refactoring --- .github/workflows/testing.yml | 4 ++-- .../connection_adapters/clickhouse/column.rb | 21 +++++++++++++++++++ .../clickhouse/schema_creation.rb | 3 +++ .../clickhouse/schema_statements.rb | 4 ++-- ...ema_definitions.rb => table_definition.rb} | 7 ++++++- .../connection_adapters/clickhouse_adapter.rb | 10 ++------- lib/clickhouse-activerecord/schema_dumper.rb | 12 ++++++----- lib/clickhouse-activerecord/version.rb | 2 +- .../1_create_some_table.rb | 10 +++++++++ spec/single/migration_spec.rb | 14 +++++++++++++ spec/single/model_spec.rb | 5 +++++ 11 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 lib/active_record/connection_adapters/clickhouse/column.rb rename lib/active_record/connection_adapters/clickhouse/{schema_definitions.rb => table_definition.rb} (94%) create mode 100644 spec/fixtures/migrations/dsl_table_with_codec/1_create_some_table.rb diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9c5c7788..2fb28d35 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -49,7 +49,7 @@ jobs: ruby-version: ${{ matrix.version.ruby }} bundler-cache: true - - run: bundle exec rspec spec/single + - run: bundle exec rspec spec/single --format progress tests_cluster: name: Testing cluster server @@ -94,4 +94,4 @@ jobs: ruby-version: ${{ matrix.version.ruby }} bundler-cache: true - - run: bundle exec rspec spec/cluster + - run: bundle exec rspec spec/cluster --format progress diff --git a/lib/active_record/connection_adapters/clickhouse/column.rb b/lib/active_record/connection_adapters/clickhouse/column.rb new file mode 100644 index 00000000..6a0e23a8 --- /dev/null +++ b/lib/active_record/connection_adapters/clickhouse/column.rb @@ -0,0 +1,21 @@ +module ActiveRecord + module ConnectionAdapters + module Clickhouse + class Column < ActiveRecord::ConnectionAdapters::Column + + attr_reader :codec + + def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, codec: nil, **args) + super + @codec = codec + end + + private + + def deduplicated + self + end + end + end + end +end diff --git a/lib/active_record/connection_adapters/clickhouse/schema_creation.rb b/lib/active_record/connection_adapters/clickhouse/schema_creation.rb index d9da9b8c..4af0d4e7 100644 --- a/lib/active_record/connection_adapters/clickhouse/schema_creation.rb +++ b/lib/active_record/connection_adapters/clickhouse/schema_creation.rb @@ -39,6 +39,9 @@ def add_column_options!(sql, options) if options[:map] == true sql.gsub!(/\s+(.*)/, ' Map(String, \1)') end + if options[:codec] + sql.gsub!(/\s+(.*)/, " \\1 CODEC(#{options[:codec]})") + end sql.gsub!(/(\sString)\(\d+\)/, '\1') sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options) sql diff --git a/lib/active_record/connection_adapters/clickhouse/schema_statements.rb b/lib/active_record/connection_adapters/clickhouse/schema_statements.rb index 1eb2e12e..a4e73fff 100644 --- a/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +++ b/lib/active_record/connection_adapters/clickhouse/schema_statements.rb @@ -8,7 +8,7 @@ module Clickhouse module SchemaStatements DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze - DB_EXCEPTION_REGEXP = /\ACode: \d.+\. DB::Exception:/.freeze + DB_EXCEPTION_REGEXP = /\ACode:\s+\d+\.\s+DB::Exception:/.freeze def execute(sql, name = nil, settings: {}) do_execute(sql, name, settings: settings) @@ -225,7 +225,7 @@ def new_column_from_field(table_name, field, _definitions) default_value = extract_value_from_default(field[3], field[2]) default_function = extract_default_function(field[3]) default_value = lookup_cast_type(sql_type).cast(default_value) - ClickhouseColumn.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function) + Clickhouse::Column.new(field[0], default_value, type_metadata, field[1].include?('Nullable'), default_function, codec: field[5].presence) end protected diff --git a/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb b/lib/active_record/connection_adapters/clickhouse/table_definition.rb similarity index 94% rename from lib/active_record/connection_adapters/clickhouse/schema_definitions.rb rename to lib/active_record/connection_adapters/clickhouse/table_definition.rb index c1a6335d..a9000cc3 100644 --- a/lib/active_record/connection_adapters/clickhouse/schema_definitions.rb +++ b/lib/active_record/connection_adapters/clickhouse/table_definition.rb @@ -94,10 +94,15 @@ def enum(*args, **options) args.each { |name| column(name, kind, **options.except(:limit)) } end + def column(name, type, index: nil, **options) + options[:null] = false if type.match?(/Nullable\([^)]+\)/) + super(name, type, index: index, **options) + end + private def valid_column_definition_options - super + [:array, :low_cardinality, :fixed_string, :value, :type, :map] + super + [:array, :low_cardinality, :fixed_string, :value, :type, :map, :codec] end end diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index 54c5daaa..bed0e57e 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -12,10 +12,11 @@ require 'active_record/connection_adapters/clickhouse/oid/big_integer' require 'active_record/connection_adapters/clickhouse/oid/map' require 'active_record/connection_adapters/clickhouse/oid/uuid' +require 'active_record/connection_adapters/clickhouse/column' require 'active_record/connection_adapters/clickhouse/quoting' -require 'active_record/connection_adapters/clickhouse/schema_definitions' require 'active_record/connection_adapters/clickhouse/schema_creation' require 'active_record/connection_adapters/clickhouse/schema_statements' +require 'active_record/connection_adapters/clickhouse/table_definition' require 'net/http' require 'openssl' @@ -77,13 +78,6 @@ module ConnectionAdapters register "clickhouse", "ActiveRecord::ConnectionAdapters::ClickhouseAdapter", "active_record/connection_adapters/clickhouse_adapter" end - class ClickhouseColumn < Column - private - def deduplicated - self - end - end - class ClickhouseAdapter < AbstractAdapter include Clickhouse::Quoting diff --git a/lib/clickhouse-activerecord/schema_dumper.rb b/lib/clickhouse-activerecord/schema_dumper.rb index 03d647dd..e2b34250 100644 --- a/lib/clickhouse-activerecord/schema_dumper.rb +++ b/lib/clickhouse-activerecord/schema_dumper.rb @@ -35,7 +35,7 @@ def table(table, stream) # super(table.gsub(/^\.inner\./, ''), stream) # detect view table - view_match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW\s+\S+\s+(TO (\S+))?/) + view_match = sql.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW\s+\S+\s+(?:TO (\S+))?/) end # Copy from original dumper @@ -52,7 +52,7 @@ def table(table, stream) # Add materialize flag tbl.print ', view: true' if view_match tbl.print ', materialized: true' if view_match && view_match[1].presence - tbl.print ", to: \"#{view_match[3]}\"" if view_match && view_match[3].presence + tbl.print ", to: \"#{view_match[2]}\"" if view_match && view_match[2].presence end if (id = columns.detect { |c| c.name == 'id' }) @@ -145,7 +145,7 @@ def schema_unsigned(column) end def schema_array(column) - (column.sql_type =~ /Array?\(/).nil? ? nil : true + (column.sql_type =~ /Array\(/).nil? ? nil : true end def schema_map(column) @@ -153,13 +153,14 @@ def schema_map(column) return :array end - (column.sql_type =~ /Map?\(/).nil? ? nil : true + (column.sql_type =~ /Map\(/).nil? ? nil : true end def schema_low_cardinality(column) - (column.sql_type =~ /LowCardinality?\(/).nil? ? nil : true + (column.sql_type =~ /LowCardinality\(/).nil? ? nil : true end + # @param [ActiveRecord::ConnectionAdapters::Clickhouse::Column] column def prepare_column_options(column) spec = {} spec[:unsigned] = schema_unsigned(column) @@ -169,6 +170,7 @@ def prepare_column_options(column) spec[:array] = nil end spec[:low_cardinality] = schema_low_cardinality(column) + spec[:codec] = column.codec.inspect if column.codec spec.merge(super).compact end diff --git a/lib/clickhouse-activerecord/version.rb b/lib/clickhouse-activerecord/version.rb index 00f5c2e5..46f2ea16 100644 --- a/lib/clickhouse-activerecord/version.rb +++ b/lib/clickhouse-activerecord/version.rb @@ -1,3 +1,3 @@ module ClickhouseActiverecord - VERSION = '1.1.3' + VERSION = '1.2.0' end diff --git a/spec/fixtures/migrations/dsl_table_with_codec/1_create_some_table.rb b/spec/fixtures/migrations/dsl_table_with_codec/1_create_some_table.rb new file mode 100644 index 00000000..8f331c71 --- /dev/null +++ b/spec/fixtures/migrations/dsl_table_with_codec/1_create_some_table.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CreateSomeTable < ActiveRecord::Migration[7.1] + def up + create_table :some, id: false, force: true do |t| + t.column :custom, "Nullable(UInt64) CODEC(T64, LZ4)" + end + end +end + diff --git a/spec/single/migration_spec.rb b/spec/single/migration_spec.rb index 9d92b8f6..d4c11c81 100644 --- a/spec/single/migration_spec.rb +++ b/spec/single/migration_spec.rb @@ -130,6 +130,20 @@ end end + context 'codec' do + let(:directory) { 'dsl_table_with_codec' } + it 'creates a table with custom column' do + subject + + current_schema = schema(model) + + expect(current_schema.keys.count).to eq(1) + expect(current_schema).to have_key('custom') + expect(current_schema['custom'].sql_type).to eq('Nullable(UInt64)') + expect(current_schema['custom'].codec).to eq('T64, LZ4') + end + end + context 'datetime' do let(:directory) { 'dsl_table_with_datetime_creation' } it 'creates a table with datetime columns' do diff --git a/spec/single/model_spec.rb b/spec/single/model_spec.rb index 432ca89b..d32b3320 100644 --- a/spec/single/model_spec.rb +++ b/spec/single/model_spec.rb @@ -31,6 +31,11 @@ class ModelPk < ActiveRecord::Base end end + it 'DB::Exception in row value' do + Model.create!(event_name: 'DB::Exception') + expect(Model.first.event_name).to eq('DB::Exception') + end + describe '#do_execute' do it 'returns formatted result' do result = Model.connection.do_execute('SELECT 1 AS t')