Merge pull request #20 from makandra/tk/improve-negate-performance

Improve negate performance
makandra · Oct 31, 2024 · 9f655fb · 9f655fb
2 parents b4f4a27 + d566333
commit 9f655fb
Show file tree

Hide file tree

Showing 20 changed files with 47 additions and 22 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,10 +3,10 @@ name: Tests
 'on':
   push:
     branches:
-    - master
+    - main
   pull_request:
     branches:
-    - master
+    - main
 jobs:
   test_mysql:
     runs-on: ubuntu-20.04

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,12 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 
 ### Compatible changes
 
+## 0.11.2 2024-10-31
+
+### Compatible changes
+
+- Fix: Performance of queries using a negation ("-xxx") is improved by using an anti-join instead of a "NOT IN".
+
 ## 0.11.1 2024-08-22
 
 ### Compatible changes

diff --git a/Gemfile.4.2.mysql2.lock b/Gemfile.4.2.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.4.2.pg.lock b/Gemfile.4.2.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.5.2.mysql2.lock b/Gemfile.5.2.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.5.2.pg.lock b/Gemfile.5.2.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.6.0.mysql2.lock b/Gemfile.6.0.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.6.0.pg.lock b/Gemfile.6.0.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.6.1.mysql2.lock b/Gemfile.6.1.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.6.1.pg.lock b/Gemfile.6.1.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.0.mysql2.lock b/Gemfile.7.0.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.0.pg.lock b/Gemfile.7.0.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.1.mysql2.lock b/Gemfile.7.1.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.1.pg.lock b/Gemfile.7.1.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.2.mysql2.lock b/Gemfile.7.2.mysql2.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/Gemfile.7.2.pg.lock b/Gemfile.7.2.pg.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minidusen (0.11.1)
+    minidusen (0.11.2)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
       edge_rider (>= 0.2.5)

diff --git a/lib/minidusen/syntax.rb b/lib/minidusen/syntax.rb
@@ -3,6 +3,7 @@ class Syntax
 
     def initialize
       @scopers = {}
+      @alias_count = 0
     end
 
     def learn_field(field, &scoper)
@@ -45,9 +46,19 @@ def apply_query(instance, root_scope, query)
 
     def append_excludes(instance, matches, exclude_query)
       excluded_records = apply_query(instance, matches.origin_class, exclude_query)
-      qualified_id_field = Util.qualify_column_name(excluded_records, excluded_records.primary_key)
-      exclude_sql = "#{qualified_id_field} NOT IN (#{excluded_records.select(qualified_id_field).to_sql})"
-      matches.where(exclude_sql)
+      primary_key = excluded_records.primary_key
+      join_alias = "exclude_#{@alias_count += 1}"
+      # due to performance reasons on big tables this needs to be implemented as an anti-join
+      # will generate SQL like
+      # LEFT JOIN (SELECT "users"."id" FROM "users" WHERE $condition) excluded
+      # ON "users"."id" = "excluded"."id"
+      # WHERE "excluded"."id" IS NULL
+      matches
+        .joins(<<~SQL)
+          LEFT JOIN (#{excluded_records.select(primary_key).to_sql}) #{join_alias}
+          ON #{Util.qualify_column_name(excluded_records, primary_key)} = #{Util.qualify_column_name(excluded_records, primary_key, table_name: join_alias)}
+        SQL
+        .where(join_alias => { primary_key => nil })
     end
 
   end

diff --git a/lib/minidusen/util.rb b/lib/minidusen/util.rb
@@ -41,10 +41,10 @@ def escape_for_like_query(phrase)
       escape_with_backslash(phrase, ['%', '_'])
     end
 
-    def qualify_column_name(model, column_name)
+    def qualify_column_name(model, column_name, table_name: model.table_name)
       column_name = column_name.to_s
       unless column_name.include?('.')
-        quoted_table_name = model.connection.quote_table_name(model.table_name)
+        quoted_table_name = model.connection.quote_table_name(table_name)
         quoted_column_name = model.connection.quote_column_name(column_name)
         column_name = "#{quoted_table_name}.#{quoted_column_name}"
       end

diff --git a/lib/minidusen/version.rb b/lib/minidusen/version.rb
@@ -1,3 +1,3 @@
 module Minidusen
-  VERSION = '0.11.1'
+  VERSION = '0.11.2'
 end
diff --git a/spec/minidusen/filter_spec.rb b/spec/minidusen/filter_spec.rb
@@ -165,6 +165,14 @@
         user_filter.filter(User, '-name_and_city_regex:Power').to_a.should == [match]
       end
 
+      it 'can be filtered twice' do
+        match = User.create!(:name => 'Sunny Flower', :city => "Flower")
+        no_match = User.create!(:name => 'Sunny Power', :city => "Power")
+        also_no_match = User.create!(:name => 'Sunny Forever', :city => "Forever")
+        first_result = user_filter.filter(User, '-name_and_city_regex:Power')
+        user_filter.filter(first_result, '-name_and_city_regex:Forever').to_a.should == [match]
+      end
+
     end
 
     context 'when the given query is blank' do