diff --git a/.docker/clickhouse/cluster/server1_config.xml b/.docker/clickhouse/cluster/server1_config.xml new file mode 100644 index 00000000..ecebb8c3 --- /dev/null +++ b/.docker/clickhouse/cluster/server1_config.xml @@ -0,0 +1,117 @@ + + + + 8123 + 9009 + clickhouse1 + + users.xml + default + default + + 5368709120 + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /var/lib/clickhouse/access/ + 3 + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + 1 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + 10000 + + + + + 1 + clickhouse1 + 9000 + + + 2 + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9181 + + + clickhouse2 + 9181 + + + + + test_cluster + clickhouse1 + 1 + + + + /clickhouse/test_cluster/task_queue/ddl + + + + system + query_log
+ toYYYYMM(event_date) + 1000 + + + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + accept, origin, x-requested-with, content-type, authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+ + diff --git a/.docker/clickhouse/cluster/server2_config.xml b/.docker/clickhouse/cluster/server2_config.xml new file mode 100644 index 00000000..83d7bbb1 --- /dev/null +++ b/.docker/clickhouse/cluster/server2_config.xml @@ -0,0 +1,117 @@ + + + + 8123 + 9009 + clickhouse2 + + users.xml + default + default + + 5368709120 + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /var/lib/clickhouse/access/ + 3 + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + 1 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + 10000 + + + + + 1 + clickhouse1 + 9000 + + + 2 + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9181 + + + clickhouse2 + 9181 + + + + + test_cluster + clickhouse2 + 1 + + + + /clickhouse/test_cluster/task_queue/ddl + + + + system + query_log
+ toYYYYMM(event_date) + 1000 + + + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + accept, origin, x-requested-with, content-type, authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+ + diff --git a/.docker/clickhouse/single/config.xml b/.docker/clickhouse/single/config.xml new file mode 100644 index 00000000..218229cd --- /dev/null +++ b/.docker/clickhouse/single/config.xml @@ -0,0 +1,54 @@ + + + + 8123 + 9000 + + users.xml + default + default + + 5368709120 + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /var/lib/clickhouse/access/ + 3 + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + 1 + + + + system + query_log
+ toYYYYMM(event_date) + 1000 + + + +
+ Access-Control-Allow-Origin + * +
+
+ Access-Control-Allow-Headers + accept, origin, x-requested-with, content-type, authorization +
+
+ Access-Control-Allow-Methods + POST, GET, OPTIONS +
+
+ Access-Control-Max-Age + 86400 +
+ + + diff --git a/.docker/clickhouse/users.xml b/.docker/clickhouse/users.xml new file mode 100644 index 00000000..61188536 --- /dev/null +++ b/.docker/clickhouse/users.xml @@ -0,0 +1,34 @@ + + + + + + random + + + + + + + + ::/0 + + default + default + 1 + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/.docker/docker-compose.cluster.yml b/.docker/docker-compose.cluster.yml new file mode 100644 index 00000000..dd59ccd4 --- /dev/null +++ b/.docker/docker-compose.cluster.yml @@ -0,0 +1,41 @@ +version: '3.5' + +services: + clickhouse1: + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}' + ulimits: + nofile: + soft: 262144 + hard: 262144 + hostname: clickhouse1 + container_name: clickhouse-activerecord-clickhouse-server-1 + ports: + - '8124:8123' + - '9001:9000' + volumes: + - './clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml' + - './clickhouse/users.xml:/etc/clickhouse-server/users.xml' + + clickhouse2: + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}' + ulimits: + nofile: + soft: 262144 + hard: 262144 + hostname: clickhouse2 + container_name: clickhouse-activerecord-clickhouse-server-2 + ports: + - '8125:8123' + volumes: + - './clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml' + - './clickhouse/users.xml:/etc/clickhouse-server/users.xml' + + # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests + nginx: + image: 'nginx:1.23.1-alpine' + hostname: nginx + ports: + - '28123:8123' + volumes: + - './nginx/local.conf:/etc/nginx/conf.d/local.conf' + container_name: clickhouse-activerecord-nginx diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml new file mode 100644 index 00000000..1176e95c --- /dev/null +++ b/.docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' +services: + clickhouse: + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}' + container_name: 'clickhouse-activerecord-clickhouse-server' + ports: + - '18123:8123' + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - './clickhouse/single/config.xml:/etc/clickhouse-server/config.xml' + - './clickhouse/users.xml:/etc/clickhouse-server/users.xml' diff --git a/.docker/nginx/local.conf b/.docker/nginx/local.conf new file mode 100644 index 00000000..35fd4512 --- /dev/null +++ b/.docker/nginx/local.conf @@ -0,0 +1,12 @@ +upstream clickhouse_cluster { + server clickhouse1:8123; + server clickhouse2:8123; +} + +server { + listen 8123; + client_max_body_size 100M; + location / { + proxy_pass http://clickhouse_cluster; + } +} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 00000000..d42e8e02 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,75 @@ +name: Testing + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + tests_single: + name: Testing single server + runs-on: ubuntu-latest + + env: + CLICKHOUSE_PORT: 18123 + CLICKHOUSE_DATABASE: default + + strategy: + fail-fast: true + matrix: + ruby-version: [ '3.0' ] + clickhouse: [ '22.1' ] + + steps: + - uses: actions/checkout@v4 + + - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker + uses: isbang/compose-action@v1.5.1 + env: + CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} + with: + compose-file: '.docker/docker-compose.yml' + down-flags: '--volumes' + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - run: bundle exec rspec spec/single + + tests_cluster: + name: Testing cluster server + runs-on: ubuntu-latest + + env: + CLICKHOUSE_PORT: 28123 + CLICKHOUSE_DATABASE: default + CLICKHOUSE_CLUSTER: test_cluster + + strategy: + fail-fast: true + matrix: + ruby-version: [ '3.0' ] + clickhouse: [ '22.1' ] + + steps: + - uses: actions/checkout@v4 + + - name: Start ClickHouse Cluster (version - ${{ matrix.clickhouse }}) in Docker + uses: isbang/compose-action@v1.5.1 + env: + CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} + with: + compose-file: '.docker/docker-compose.cluster.yml' + down-flags: '--volumes' + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - run: bundle exec rspec spec/cluster diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbfb857..02636c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ +### Version 1.0.5 (Mar 14, 2024) + +* GitHub workflows +* Fix injection internal and schema classes for rails 7 +* Add support for binary string by [@PauloMiranda98](https://github.com/PauloMiranda98) in (#116) + +### Version 1.0.4 (Feb 2, 2024) + +* Use ILIKE for `model.arel_table[:column]#matches` by [@stympy](https://github.com/stympy) in (#115) +* Fixed `insert_all` for array column (#71) +* Register Bool and UUID in type map by [@lukinski](https://github.com/lukinski) in (#110) +* Refactoring `final` method +* Support update & delete for clickhouse from version 23.3 and newer (#93) + +### Version 1.0.0 (Nov 29, 2023) + + * Full support Rails 7.1+ + * Full support primary or multiple databases + +### Version 0.6.0 (Oct 19, 2023) + + * Added `Bool` column type instead `Uint8` (#78). Supports ClickHouse 22+ database only + * Added `final` method (#81) (The `ar_internal_metadata` table needs to be deleted after a gem update) + * Added `settings` method (#82) + * Fixed convert aggregation type (#92) + * Fixed raise error not database exist (#91) + * Fixed internal metadata update (#84) + ### Version 0.5.10 (Jun 22, 2022) * Fixes to create_table method (#70) diff --git a/lib/active_record/connection_adapters/clickhouse/schema_statements.rb b/lib/active_record/connection_adapters/clickhouse/schema_statements.rb index bb93ff45..d7a3386d 100644 --- a/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +++ b/lib/active_record/connection_adapters/clickhouse/schema_statements.rb @@ -35,13 +35,20 @@ def exec_insert_all(sql, name) # @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update def exec_update(_sql, _name = nil, _binds = []) do_execute(_sql, _name, format: nil) - true + 0 end # @link https://clickhouse.com/docs/en/sql-reference/statements/delete def exec_delete(_sql, _name = nil, _binds = []) - do_execute(_sql, _name, format: nil) - true + log(_sql, "#{adapter_name} #{_name}") do + res = request(_sql) + begin + data = JSON.parse(res.header['x-clickhouse-summary']) + data['result_rows'].to_i + rescue JSONError + 0 + end + end end def tables(name = nil) @@ -66,18 +73,14 @@ def data_sources def do_system_execute(sql, name = nil) log_with_debug(sql, "#{adapter_name} #{name}") do - res = @connection.post("/?#{@connection_config.to_param}", "#{sql} FORMAT #{DEFAULT_RESPONSE_FORMAT}", 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}") - + res = request(sql, DEFAULT_RESPONSE_FORMAT) process_response(res, DEFAULT_RESPONSE_FORMAT) end end def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {}) log(sql, "#{adapter_name} #{name}") do - formatted_sql = apply_format(sql, format) - request_params = @connection_config || {} - res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}") - + res = request(sql, format, settings) process_response(res, format) end end @@ -114,6 +117,17 @@ def with_yaml_fallback(value) # :nodoc: private + # Make HTTP request to ClickHouse server + # @param [String] sql + # @param [String, nil] format + # @param [Hash] settings + # @return [Net::HTTPResponse] + def request(sql, format = nil, settings = {}) + formatted_sql = apply_format(sql, format) + request_params = @connection_config || {} + @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}") + end + def apply_format(sql, format) format ? "#{sql} FORMAT #{format}" : sql end @@ -170,8 +184,7 @@ def table_structure(table_name) return data unless data.empty? - raise ActiveRecord::StatementInvalid, - "Could not find table '#{table_name}'" + raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'" end alias column_definitions table_structure diff --git a/lib/active_record/connection_adapters/clickhouse_adapter.rb b/lib/active_record/connection_adapters/clickhouse_adapter.rb index 8b8ce4a6..76f85d80 100644 --- a/lib/active_record/connection_adapters/clickhouse_adapter.rb +++ b/lib/active_record/connection_adapters/clickhouse_adapter.rb @@ -73,10 +73,11 @@ def is_view def is_view=(value) @is_view = value end - # - # def arel_table # :nodoc: - # @arel_table ||= Arel::Table.new(table_name, type_caster: type_caster) - # end + + def _delete_record(constraints) + raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key + super + end end end diff --git a/lib/arel/visitors/clickhouse.rb b/lib/arel/visitors/clickhouse.rb index f39942d4..a8fd5993 100644 --- a/lib/arel/visitors/clickhouse.rb +++ b/lib/arel/visitors/clickhouse.rb @@ -13,6 +13,13 @@ def aggregate(name, o, collector) end end + # https://clickhouse.com/docs/en/sql-reference/statements/delete + # DELETE and UPDATE in ClickHouse working only without table name + def visit_Arel_Attributes_Attribute(o, collector) + collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ') + collector << quote_column_name(o.name) + end + def visit_Arel_Nodes_SelectOptions(o, collector) maybe_visit o.settings, super end diff --git a/lib/clickhouse-activerecord.rb b/lib/clickhouse-activerecord.rb index 67f95bda..4eeb4158 100644 --- a/lib/clickhouse-activerecord.rb +++ b/lib/clickhouse-activerecord.rb @@ -24,9 +24,9 @@ module ClickhouseActiverecord def self.load - ActiveRecord::InternalMetadata.singleton_class.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods) + ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods) ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation) - ActiveRecord::SchemaMigration.singleton_class.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods) + ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods) Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore) Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement) diff --git a/lib/clickhouse-activerecord/version.rb b/lib/clickhouse-activerecord/version.rb index 3d160dec..a03d7693 100644 --- a/lib/clickhouse-activerecord/version.rb +++ b/lib/clickhouse-activerecord/version.rb @@ -1,3 +1,3 @@ module ClickhouseActiverecord - VERSION = '1.0.4' + VERSION = '1.0.5' end diff --git a/lib/core_extensions/active_record/internal_metadata.rb b/lib/core_extensions/active_record/internal_metadata.rb index 79f41b79..85b28e24 100644 --- a/lib/core_extensions/active_record/internal_metadata.rb +++ b/lib/core_extensions/active_record/internal_metadata.rb @@ -3,17 +3,6 @@ module ActiveRecord module InternalMetadata module ClassMethods - def []=(key, value) - row = final.find_by(key: key) - if row.nil? || row.value != value - create!(key: key, value: value) - end - end - - def [](key) - final.where(key: key).pluck(:value).first - end - def create_table return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) return if table_exists? || !enabled? @@ -21,7 +10,7 @@ def create_table key_options = connection.internal_string_options_for_primary_key table_options = { id: false, - options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '', + options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key', if_not_exists: true } full_config = connection.instance_variable_get(:@config) || {} @@ -40,6 +29,27 @@ def create_table t.timestamps end end + + private + + def update_entry(key, new_value) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + create_entry(key, new_value) + end + + def select_entry(key) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + sm = ::Arel::SelectManager.new(arel_table) + sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/ + sm.project(::Arel.star) + sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key))) + sm.order(arel_table[primary_key].asc) + sm.limit = 1 + + connection.select_one(sm, "#{self.class} Load") + end end end end diff --git a/lib/core_extensions/active_record/schema_migration.rb b/lib/core_extensions/active_record/schema_migration.rb index a6de4ffb..6f36ad97 100644 --- a/lib/core_extensions/active_record/schema_migration.rb +++ b/lib/core_extensions/active_record/schema_migration.rb @@ -32,7 +32,7 @@ def create_table def delete_version(version) return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) - im = Arel::InsertManager.new(arel_table) + im = ::Arel::InsertManager.new(arel_table) im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) end diff --git a/spec/cluster/migration_spec.rb b/spec/cluster/migration_spec.rb new file mode 100644 index 00000000..59782e22 --- /dev/null +++ b/spec/cluster/migration_spec.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +RSpec.describe 'Cluster Migration', :migrations do + describe 'performs migrations' do + let(:model) do + Class.new(ActiveRecord::Base) do + self.table_name = 'some' + end + end + let(:directory) { raise 'NotImplemented' } + let(:migrations_dir) { File.join(FIXTURES_PATH, 'migrations', directory) } + let(:migration_context) { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration, model.connection.internal_metadata) } + + connection_config = ActiveRecord::Base.connection_db_config.configuration_hash + + before(:all) do + raise 'Unknown cluster name in config' if connection_config[:cluster_name].blank? + end + + subject do + quietly { migration_context.up } + end + + context 'dsl' do + context 'with distributed' do + let(:model_distributed) do + Class.new(ActiveRecord::Base) do + self.table_name = 'some_distributed' + end + end + + let(:directory) { 'dsl_create_table_with_distributed' } + it 'creates a table with distributed table' do + subject + + current_schema = schema(model) + current_schema_distributed = schema(model_distributed) + + expect(current_schema.keys.count).to eq(1) + expect(current_schema_distributed.keys.count).to eq(1) + + expect(current_schema).to have_key('date') + expect(current_schema_distributed).to have_key('date') + + expect(current_schema['date'].sql_type).to eq('Date') + expect(current_schema_distributed['date'].sql_type).to eq('Date') + end + + it 'drops a table with distributed table' do + subject + + expect(ActiveRecord::Base.connection.tables).to include('some') + expect(ActiveRecord::Base.connection.tables).to include('some_distributed') + + quietly do + migration_context.down + end + + expect(ActiveRecord::Base.connection.tables).not_to include('some') + expect(ActiveRecord::Base.connection.tables).not_to include('some_distributed') + end + end + end + + context 'with alias in cluster_name' do + let(:model) do + Class.new(ActiveRecord::Base) do + self.table_name = 'some' + end + end + connection_config = ActiveRecord::Base.connection_db_config.configuration_hash + + before(:all) do + ActiveRecord::Base.establish_connection(connection_config.merge(cluster_name: '{cluster}')) + end + + after(:all) do + ActiveRecord::Base.establish_connection(connection_config) + end + + let(:directory) { 'dsl_create_table_with_cluster_name_alias' } + it 'creates a table' do + subject + + current_schema = schema(model) + + expect(current_schema.keys.count).to eq(1) + expect(current_schema).to have_key('date') + expect(current_schema['date'].sql_type).to eq('Date') + end + + it 'drops a table' do + subject + + expect(ActiveRecord::Base.connection.tables).to include('some') + + # Need for sync between clickhouse servers + ActiveRecord::Base.connection.execute('SELECT * FROM schema_migrations') + + quietly do + migration_context.down + end + + expect(ActiveRecord::Base.connection.tables).not_to include('some') + end + end + end +end diff --git a/spec/fixtures/migrations/add_sample_data/2_create_join_table.rb b/spec/fixtures/migrations/add_sample_data/2_create_join_table.rb index bfbacbcd..7f4fb543 100644 --- a/spec/fixtures/migrations/add_sample_data/2_create_join_table.rb +++ b/spec/fixtures/migrations/add_sample_data/2_create_join_table.rb @@ -4,6 +4,7 @@ class CreateJoinTable < ActiveRecord::Migration[5.0] def up create_table :joins, options: 'MergeTree PARTITION BY toYYYYMM(date) ORDER BY (event_name)' do |t| t.string :event_name, null: false + t.integer :event_value t.integer :join_value t.date :date, null: false end diff --git a/spec/fixtures/migrations/dsl_create_table_with_cluster_name_alias/1_create_some_table.rb b/spec/fixtures/migrations/dsl_create_table_with_cluster_name_alias/1_create_some_table.rb index 45f38644..0fc454df 100644 --- a/spec/fixtures/migrations/dsl_create_table_with_cluster_name_alias/1_create_some_table.rb +++ b/spec/fixtures/migrations/dsl_create_table_with_cluster_name_alias/1_create_some_table.rb @@ -1,6 +1,6 @@ class CreateSomeTable < ActiveRecord::Migration[5.0] def change - create_table :some, options: 'MergeTree PARTITION BY toYYYYMM(date) ORDER BY (date)' do |t| + create_table :some, options: 'MergeTree PARTITION BY toYYYYMM(date) ORDER BY (date)', sync: true, id: false do |t| t.date :date, null: false end end diff --git a/spec/cases/migration_spec.rb b/spec/single/migration_spec.rb similarity index 74% rename from spec/cases/migration_spec.rb rename to spec/single/migration_spec.rb index 9f2de6cc..163452b8 100644 --- a/spec/cases/migration_spec.rb +++ b/spec/single/migration_spec.rb @@ -11,11 +11,7 @@ let(:migrations_dir) { File.join(FIXTURES_PATH, 'migrations', directory) } let(:migration_context) { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration, model.connection.internal_metadata) } - if ActiveRecord::version >= Gem::Version.new('6.1') - connection_config = ActiveRecord::Base.connection_db_config.configuration_hash - else - connection_config = ActiveRecord::Base.connection_config - end + connection_config = ActiveRecord::Base.connection_db_config.configuration_hash subject do quietly { migration_context.up } @@ -190,53 +186,6 @@ end end - context 'with distributed' do - let(:model_distributed) do - Class.new(ActiveRecord::Base) do - self.table_name = 'some_distributed' - end - end - - before(:all) do - ActiveRecord::Base.establish_connection(connection_config.merge(cluster_name: CLUSTER_NAME)) - end - - after(:all) do - ActiveRecord::Base.establish_connection(connection_config) - end - - let(:directory) { 'dsl_create_table_with_distributed' } - it 'creates a table with distributed table' do - subject - - current_schema = schema(model) - current_schema_distributed = schema(model_distributed) - - expect(current_schema.keys.count).to eq(1) - expect(current_schema_distributed.keys.count).to eq(1) - - expect(current_schema).to have_key('date') - expect(current_schema_distributed).to have_key('date') - - expect(current_schema['date'].sql_type).to eq('Date') - expect(current_schema_distributed['date'].sql_type).to eq('Date') - end - - it 'drops a table with distributed table' do - subject - - expect(ActiveRecord::Base.connection.tables).to include('some') - expect(ActiveRecord::Base.connection.tables).to include('some_distributed') - - quietly do - migration_context.down - end - - expect(ActiveRecord::Base.connection.tables).not_to include('some') - expect(ActiveRecord::Base.connection.tables).not_to include('some_distributed') - end - end - context 'creates a view' do let(:directory) { 'dsl_create_view_with_to_section' } it 'creates a view' do @@ -261,50 +210,6 @@ end end end - - context 'with alias in cluster_name' do - let(:model) do - Class.new(ActiveRecord::Base) do - self.table_name = 'some' - end - end - if ActiveRecord::version >= Gem::Version.new('6.1') - connection_config = ActiveRecord::Base.connection_db_config.configuration_hash - else - connection_config = ActiveRecord::Base.connection_config - end - - before(:all) do - ActiveRecord::Base.establish_connection(connection_config.merge(cluster_name: '{cluster}')) - end - - after(:all) do - ActiveRecord::Base.establish_connection(connection_config) - end - - let(:directory) { 'dsl_create_table_with_cluster_name_alias' } - it 'creates a table' do - subject - - current_schema = schema(model) - - expect(current_schema.keys.count).to eq(1) - expect(current_schema).to have_key('date') - expect(current_schema['date'].sql_type).to eq('Date') - end - - it 'drops a table' do - subject - - expect(ActiveRecord::Base.connection.tables).to include('some') - - quietly do - migration_context.down - end - - expect(ActiveRecord::Base.connection.tables).not_to include('some') - end - end end describe 'drop table' do diff --git a/spec/cases/model_spec.rb b/spec/single/model_spec.rb similarity index 64% rename from spec/cases/model_spec.rb rename to spec/single/model_spec.rb index 165afbae..d59bc80b 100644 --- a/spec/cases/model_spec.rb +++ b/spec/single/model_spec.rb @@ -2,36 +2,38 @@ RSpec.describe 'Model', :migrations do + class ModelJoin < ActiveRecord::Base + self.table_name = 'joins' + belongs_to :model, class_name: 'Model' + end + class Model < ActiveRecord::Base + self.table_name = 'sample' + has_many :joins, class_name: 'ModelJoin', primary_key: 'event_name' + end + class ModelPk < ActiveRecord::Base + self.table_name = 'sample' + self.primary_key = 'event_name' + end + let(:date) { Date.today } context 'sample' do - let!(:model) do - class ModelJoin < ActiveRecord::Base - self.table_name = 'joins' - belongs_to :model, class_name: 'Model' - end - class Model < ActiveRecord::Base - self.table_name = 'sample' - has_many :joins, class_name: 'ModelJoin', primary_key: 'event_name' - end - Model - end before do migrations_dir = File.join(FIXTURES_PATH, 'migrations', 'add_sample_data') - quietly { ActiveRecord::MigrationContext.new(migrations_dir, model.connection.schema_migration).up } + quietly { ActiveRecord::MigrationContext.new(migrations_dir, Model.connection.schema_migration).up } end describe '#do_execute' do it 'returns formatted result' do - result = model.connection.do_execute('SELECT 1 AS t') + result = Model.connection.do_execute('SELECT 1 AS t') expect(result['data']).to eq([[1]]) expect(result['meta']).to eq([{ 'name' => 't', 'type' => 'UInt8' }]) end context 'with JSONCompact format' do it 'returns formatted result' do - result = model.connection.do_execute('SELECT 1 AS t', format: 'JSONCompact') + result = Model.connection.do_execute('SELECT 1 AS t', format: 'JSONCompact') expect(result['data']).to eq([[1]]) expect(result['meta']).to eq([{ 'name' => 't', 'type' => 'UInt8' }]) end @@ -39,7 +41,7 @@ class Model < ActiveRecord::Base context 'with JSONCompactEachRowWithNamesAndTypes format' do it 'returns formatted result' do - result = model.connection.do_execute('SELECT 1 AS t', format: 'JSONCompactEachRowWithNamesAndTypes') + result = Model.connection.do_execute('SELECT 1 AS t', format: 'JSONCompactEachRowWithNamesAndTypes') expect(result['data']).to eq([[1]]) expect(result['meta']).to eq([{ 'name' => 't', 'type' => 'UInt8' }]) end @@ -49,84 +51,104 @@ class Model < ActiveRecord::Base describe '#create' do it 'creates a new record' do expect { - model.create!( + Model.create!( event_name: 'some event', date: date ) - }.to change { model.count } + }.to change { Model.count } end it 'insert all' do if ActiveRecord::version >= Gem::Version.new('6') - model.insert_all([ + Model.insert_all([ {event_name: 'some event 1', date: date}, {event_name: 'some event 2', date: date}, ]) - expect(model.count).to eq(2) + expect(Model.count).to eq(2) end end end describe '#update' do - let(:record) { model.create!(event_name: 'some event', date: date) } + let!(:record) { Model.create!(event_name: 'some event', event_value: 1, date: date) } + + it 'update' do + expect { + Model.where(event_name: 'some event').update_all(event_value: 2) + }.to_not raise_error + end - it 'raises an error' do - record.update!(event_name: 'new event name') - expect(model.where(event_name: 'new event name').count).to eq(1) + it 'update model with primary key' do + expect { + ModelPk.first.update!(event_value: 2) + }.to_not raise_error end end - describe '#destroy' do - let(:record) { model.create!(event_name: 'some event', date: date) } + describe '#delete' do + let!(:record) { Model.create!(event_name: 'some event', date: date) } + + it 'model destroy' do + expect { + record.destroy! + }.to raise_error(ActiveRecord::ActiveRecordError, 'Deleting a row is not possible without a primary key') + end + + it 'scope' do + expect { + Model.where(event_name: 'some event').delete_all + }.to_not raise_error + end - it 'raises an error' do - record.destroy! - expect(model.count).to eq(0) + it 'destroy model with primary key' do + expect { + ModelPk.first.destroy! + }.to_not raise_error end end describe '#reverse_order!' do it 'blank' do - expect(model.all.reverse_order!.map(&:event_name)).to eq([]) + expect(Model.all.reverse_order!.map(&:event_name)).to eq([]) end it 'select' do - model.create!(event_name: 'some event 1', date: 1.day.ago) - model.create!(event_name: 'some event 2', date: 2.day.ago) - expect(model.all.reverse_order!.map(&:event_name)).to eq(['some event 1', 'some event 2']) + Model.create!(event_name: 'some event 1', date: 1.day.ago) + Model.create!(event_name: 'some event 2', date: 2.day.ago) + expect(Model.all.reverse_order!.map(&:event_name)).to eq(['some event 1', 'some event 2']) end end describe 'convert type with aggregations' do - let!(:record1) { model.create!(event_name: 'some event', event_value: 1, date: date) } - let!(:record2) { model.create!(event_name: 'some event', event_value: 3, date: date) } + let!(:record1) { Model.create!(event_name: 'some event', event_value: 1, date: date) } + let!(:record2) { Model.create!(event_name: 'some event', event_value: 3, date: date) } it 'integer' do - expect(model.select(Arel.sql('sum(event_value) AS event_value')).first.event_value.class).to eq(Integer) - expect(model.select(Arel.sql('sum(event_value) AS value')).first.attributes['value'].class).to eq(Integer) - expect(model.pluck(Arel.sql('sum(event_value)')).first[0].class).to eq(Integer) + expect(Model.select(Arel.sql('sum(event_value) AS event_value')).first.event_value.class).to eq(Integer) + expect(Model.select(Arel.sql('sum(event_value) AS value')).first.attributes['value'].class).to eq(Integer) + expect(Model.pluck(Arel.sql('sum(event_value)')).first[0].class).to eq(Integer) end end describe 'boolean column type' do - let!(:record1) { model.create!(event_name: 'some event', event_value: 1, date: date) } + let!(:record1) { Model.create!(event_name: 'some event', event_value: 1, date: date) } it 'bool result' do - expect(model.first.enabled.class).to eq(FalseClass) + expect(Model.first.enabled.class).to eq(FalseClass) end it 'is mapped to :boolean' do - type = model.columns_hash['enabled'].type + type = Model.columns_hash['enabled'].type expect(type).to eq(:boolean) end end describe 'string column type as byte array' do let(:bytes) { (0..255).to_a } - let!(:record1) { model.create!(event_name: 'some event', byte_array: bytes.pack('C*')) } + let!(:record1) { Model.create!(event_name: 'some event', byte_array: bytes.pack('C*')) } it 'keeps all bytes' do - returned_byte_array = model.first.byte_array + returned_byte_array = Model.first.byte_array expect(returned_byte_array.unpack('C*')).to eq(bytes) end @@ -135,11 +157,11 @@ class Model < ActiveRecord::Base describe 'UUID column type' do let(:random_uuid) { SecureRandom.uuid } let!(:record1) do - model.create!(event_name: 'some event', event_value: 1, date: date, relation_uuid: random_uuid) + Model.create!(event_name: 'some event', event_value: 1, date: date, relation_uuid: random_uuid) end it 'is mapped to :uuid' do - type = model.columns_hash['relation_uuid'].type + type = Model.columns_hash['relation_uuid'].type expect(type).to eq(:uuid) end @@ -155,37 +177,42 @@ class Model < ActiveRecord::Base describe '#settings' do it 'works' do - sql = model.settings(optimize_read_in_order: 1, cast_keep_nullable: 1).to_sql + sql = Model.settings(optimize_read_in_order: 1, cast_keep_nullable: 1).to_sql expect(sql).to eq('SELECT sample.* FROM sample SETTINGS optimize_read_in_order = 1, cast_keep_nullable = 1') end it 'quotes' do - sql = model.settings(foo: :bar).to_sql + sql = Model.settings(foo: :bar).to_sql expect(sql).to eq('SELECT sample.* FROM sample SETTINGS foo = \'bar\'') end it 'allows passing the symbol :default to reset a setting' do - sql = model.settings(max_insert_block_size: :default).to_sql + sql = Model.settings(max_insert_block_size: :default).to_sql expect(sql).to eq('SELECT sample.* FROM sample SETTINGS max_insert_block_size = DEFAULT') end end describe '#using' do it 'works' do - sql = model.joins(:joins).using(:event_name, :date).to_sql + sql = Model.joins(:joins).using(:event_name, :date).to_sql expect(sql).to eq('SELECT sample.* FROM sample INNER JOIN joins USING event_name,date') end + + it 'works with filters' do + sql = Model.joins(:joins).using(:event_name, :date).where(joins: { event_value: 1 }).to_sql + expect(sql).to eq("SELECT sample.* FROM sample INNER JOIN joins USING event_name,date WHERE joins.event_value = 1") + end end describe 'arel predicates' do describe '#matches' do it 'uses ilike for case insensitive matches' do - sql = model.where(model.arel_table[:event_name].matches('some event')).to_sql + sql = Model.where(Model.arel_table[:event_name].matches('some event')).to_sql expect(sql).to eq("SELECT sample.* FROM sample WHERE sample.event_name ILIKE 'some event'") end it 'uses like for case sensitive matches' do - sql = model.where(model.arel_table[:event_name].matches('some event', nil, true)).to_sql + sql = Model.where(Model.arel_table[:event_name].matches('some event', nil, true)).to_sql expect(sql).to eq("SELECT sample.* FROM sample WHERE sample.event_name LIKE 'some event'") end end @@ -194,8 +221,8 @@ class Model < ActiveRecord::Base describe 'DateTime64 create' do it 'create a new record' do time = DateTime.parse('2023-07-21 08:00:00.123') - model.create!(datetime: time, datetime64: time) - row = model.first + Model.create!(datetime: time, datetime64: time) + row = Model.first expect(row.datetime).to_not eq(row.datetime64) expect(row.datetime.strftime('%Y-%m-%d %H:%M:%S')).to eq('2023-07-21 08:00:00') expect(row.datetime64.strftime('%Y-%m-%d %H:%M:%S.%3N')).to eq('2023-07-21 08:00:00.123') @@ -203,14 +230,14 @@ class Model < ActiveRecord::Base end describe 'final request' do - let!(:record1) { model.create!(date: date, event_name: '1') } - let!(:record2) { model.create!(date: date, event_name: '1') } + let!(:record1) { Model.create!(date: date, event_name: '1') } + let!(:record2) { Model.create!(date: date, event_name: '1') } it 'select' do - expect(model.count).to eq(2) - expect(model.final.count).to eq(1) - expect(model.final!.count).to eq(1) - expect(model.final.where(date: '2023-07-21').to_sql).to eq('SELECT sample.* FROM sample FINAL WHERE sample.date = \'2023-07-21\'') + expect(Model.count).to eq(2) + expect(Model.final.count).to eq(1) + expect(Model.final!.count).to eq(1) + expect(Model.final.where(date: '2023-07-21').to_sql).to eq('SELECT sample.* FROM sample FINAL WHERE sample.date = \'2023-07-21\'') end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 78e2fca4..ccf95d31 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,7 +9,6 @@ ClickhouseActiverecord.load FIXTURES_PATH = File.join(File.dirname(__FILE__), 'fixtures') -CLUSTER_NAME = 'test' RSpec.configure do |config| # Enable flags like --only-failures and --next-failure @@ -38,10 +37,12 @@ default: { adapter: 'clickhouse', host: 'localhost', - port: 8123, - database: 'test', + port: ENV['CLICKHOUSE_PORT'] || 8123, + database: ENV['CLICKHOUSE_DATABASE'] || 'test', username: nil, - password: nil + password: nil, + use_metadata_table: false, + cluster_name: ENV['CLICKHOUSE_CLUSTER'], } ) @@ -55,15 +56,11 @@ def schema(model) end def clear_db - if ActiveRecord::version >= Gem::Version.new('6.1') - cluster = ActiveRecord::Base.connection_db_config.configuration_hash[:cluster_name] - else - cluster = ActiveRecord::Base.connection_config[:cluster_name] - end + cluster = ActiveRecord::Base.connection_db_config.configuration_hash[:cluster_name] pattern = if cluster normalized_cluster_name = cluster.start_with?('{') ? "'#{cluster}'" : cluster - "DROP TABLE %s ON CLUSTER #{normalized_cluster_name}" + "DROP TABLE %s ON CLUSTER #{normalized_cluster_name} SYNC" else 'DROP TABLE %s' end