From c4e3262172e0982f59cb5c6de9270673b5946bb1 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 10 Nov 2024 18:47:37 -0500
Subject: [PATCH 01/46] Initial version of DbGeneric.select()

---
 gramps/gen/db/generic.py      |  14 +++
 gramps/gen/db/select_utils.py | 225 ++++++++++++++++++++++++++++++++++
 2 files changed, 239 insertions(+)
 create mode 100644 gramps/gen/db/select_utils.py

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index 5afa3d6354..0fbb66887f 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -91,6 +91,7 @@
 from .bookmarks import DbBookmarks
 from .exceptions import DbUpgradeRequiredError, DbVersionError
 from .utils import clear_lock_file, write_lock_file
+from .select_utils import select
 
 _ = glocale.translation.gettext
 
@@ -2737,3 +2738,16 @@ def set_serializer(self, serializer_name):
             self.serializer = BlobSerializer
         elif serializer_name == "json":
             self.serializer = JSONSerializer
+
+    def select(self, table, selections=None, where=None, sort_by=None):
+        """
+        Generic function that can handle jsonpath items in a list.
+
+        table - name of table
+        selections -
+            Example: ("gender", "primary_name.suname_list[0].surname", ...)
+        sort_by - "gender"
+        where - ("and", ("handle", "=", "abc64564346"),
+                        ("gramps_id", "=", ""))
+        """
+        yield from select(self, table, selections, where, sort_by)
diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
new file mode 100644
index 0000000000..a1ba3074a6
--- /dev/null
+++ b/gramps/gen/db/select_utils.py
@@ -0,0 +1,225 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2024 Doug Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+"""
+Support functions for Generic DB select functionality
+"""
+
+import jsonpath_ng
+
+PARSE_CACHE = {}
+
+
+def select(db, table, selections, where, sort_by):
+    """
+    Top-level function for select functions.
+
+    Args:
+        db: database instance
+        table: the table name
+        selections: list of json paths
+        where: a where expression (see below)
+        sort_by: list of json_paths to sort by
+    """
+    selections = selections if selections else ["$"]
+    if sort_by is None:
+        yield from select_items(db, table, selections, where)
+    else:
+        results = list(select_items(db, table, ["$"], where))
+        for row in sorted(results, key=lambda item: sort_function(item, sort_by)):
+            yield get_items(row, selections)
+
+
+def select_items(db, table, selections, where):
+    """
+    Select items with optional where expr.
+
+    Note that we apply the where, then select the items.
+    """
+    for row in select_where(db, table, where):
+        results = get_items(row, selections)
+        yield results
+
+
+def select_where(db, table, where):
+    """
+    Select items in the table where an expression
+    is True.
+
+    We don't always want to scan the entire database
+    if we don't have to. So we chech for all exact
+    matches on what we can select on directly using
+    handle and gramps_id lookup methods.
+    """
+    if where:
+        indexed_expressions = []
+        get_indexed_fields(where, indexed_expressions)
+        if indexed_expressions:
+            handles_returned = []
+            for indexed in indexed_expressions:
+                index_field, compare, index_value = indexed
+                if index_field == "$.gramps_id":
+                    func_name = "raw_id_func"
+                elif index_field == "$.handle":
+                    func_name = "raw_func"
+                else:
+                    raise Exception("index must be handle or gramps_id")
+
+                row = db._get_table_func(table.title(), func_name)(index_value)
+                if match_where(row, where):
+                    # Don't return the same row more than once:
+                    if row["handle"] not in handles_returned:
+                        handles_returned.append(row["handle"])
+                        yield row
+            return
+
+    # Otherwise, we have to scan the whole table:
+    iter_func = db._get_table_func(table.title(), "cursor_func")
+    for row in iter_func():
+        if where:
+            if match_where(row[1], where):
+                yield row[1]
+        else:
+            yield row[1]
+
+
+def sort_function(item, sort_by):
+    """
+    Given a list of json paths, return a
+    tuple of associated values for sorting.
+    """
+    results = get_items(item, sort_by)
+    return tuple(results.values())
+
+
+def get_items(row, selections):
+    """
+    Given a list of json paths, get the
+    items from the row.
+    """
+    results = {}
+    for json_path in selections:
+        if json_path == "$":
+            results = row
+        else:
+            match = match_json_path(json_path, row)
+            if match:
+                results[json_path[2:]] = match[0].value
+    return results
+
+
+def match_json_path(json_path, row):
+    """
+    Return the matching json_path item if the given
+    json_path matches.
+    """
+    if json_path not in PARSE_CACHE:
+        PARSE_CACHE[json_path] = jsonpath_ng.parse(json_path)
+    jsonpath_expr = PARSE_CACHE[json_path]
+    match = jsonpath_expr.find(row)
+    return match
+
+def eval_expr(expr, data):
+    if isinstance(expr, str) and expr.startswith("$"):
+        match = match_json_path(expr, data)
+        if match:
+            return match[0].value
+        else:
+            return None
+
+    return expr
+
+
+def match_where(data, where):
+    """
+    Assumes all comparisons are using JSONpath
+    ("gender", "=", value)
+    """
+    if len(where) == 3:
+        lhs = eval_expr(where[0], data)
+        op = where[1].lower()
+        rhs = eval_expr(where[2], data)
+        return compare_where(lhs, op, rhs)
+    elif where[0].lower() == "and":
+        for expr in where[1]:
+            result = match_where(data, expr)
+            if not result:
+                return False
+        return True
+    elif where[0].lower() == "or":
+        for expr in where[1]:
+            result = match_where(data, expr)
+            if result:
+                return True
+        return False
+    else:
+        raise Exception("Malformed where expression: %r" % where)
+
+
+def compare_where(lhs, op, rhs):
+    """
+    Evaluate the where expression.
+    """
+    if op == "=":
+        return lhs == rhs
+    elif op == "!=":
+        return lhs != rhs
+    elif op == "<":
+        return lhs < rhs
+    elif op == ">":
+        return lhs > rhs
+    elif op == "in":
+        return lhs in rhs
+    elif op == "not in":
+        return lhs not in rhs
+    else:
+        raise Exception("Operator %r is not supported" % op)
+
+
+def get_indexed_fields(where, indexes):
+    """
+    Recursive function that accumpulates matches
+    in the mutable `indexes` list.
+
+    A match is an expression that compares "$.handle"
+    or "$.gramps_id" equal to a value.
+
+    where - a tuple expression, "or" or "and"
+    indexes - pass in a list
+
+    Example where clauses:
+
+    There are 4 cases to consider:
+    ```
+    ("$.gramps_id", "=", "I0043")
+    ("$.handle", "=", "abc23652")
+    ("and", (...))
+    ("or", (...))
+    ```
+    """
+    if len(where) == 3:
+        # A comparison expression
+        if where[0] in ["$.gramps_id", "$.handle"] and where[1] == "=":
+            indexes += [where]
+    else:
+        # Must be ("and", exprs) or ("or", exprs)
+        # Recurse on all expressions
+        for expr in where[1]:
+            get_indexed_fields(expr, indexes)

From 59751dc6b4eedf86f09882dce167a4f8940b81be Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 10 Nov 2024 20:08:43 -0500
Subject: [PATCH 02/46] Added docstrings and more operators

---
 gramps/gen/db/generic.py      | 33 +++++++++++++++++++++++++--------
 gramps/gen/db/select_utils.py | 29 ++++++++++++++++++++++-------
 2 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index 0fbb66887f..25fd9a1337 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -91,7 +91,6 @@
 from .bookmarks import DbBookmarks
 from .exceptions import DbUpgradeRequiredError, DbVersionError
 from .utils import clear_lock_file, write_lock_file
-from .select_utils import select
 
 _ = glocale.translation.gettext
 
@@ -2741,13 +2740,31 @@ def set_serializer(self, serializer_name):
 
     def select(self, table, selections=None, where=None, sort_by=None):
         """
-        Generic function that can handle jsonpath items in a list.
+        Generic implementation of select, with where and sort_by clauses.
 
-        table - name of table
-        selections -
-            Example: ("gender", "primary_name.suname_list[0].surname", ...)
-        sort_by - "gender"
-        where - ("and", ("handle", "=", "abc64564346"),
-                        ("gramps_id", "=", ""))
+        :param table: Name of table
+        :type table: str
+        :param selections: List of json-paths
+        :type selections: List[str] or tuple(str)
+        :param where: A single where-expression (see below)
+        :type where: tuple or list
+        :param sort_by: A list of expressions to sort on
+        :type where: tuple or list
+        :returns: Returns selected items from select rows, from iterator
+        :rtype: dict
+
+        Examples:
+
+        Get the gender and surname of all males, sort by gramps_id:
+        ```
+        db.select(
+            "person",
+            ["$.gender", "$.primary_name.surname_list[0].surname"],
+            where=["$.gender", "=", Person.MALE],
+            sort_by=["$.gramps_id"],
+        )
+        ```
         """
+        from gramps.gen.db.select_utils import select
+
         yield from select(self, table, selections, where, sort_by)
diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
index a1ba3074a6..93b165bc2a 100644
--- a/gramps/gen/db/select_utils.py
+++ b/gramps/gen/db/select_utils.py
@@ -22,6 +22,7 @@
 Support functions for Generic DB select functionality
 """
 
+import re
 import jsonpath_ng
 
 PARSE_CACHE = {}
@@ -34,9 +35,9 @@ def select(db, table, selections, where, sort_by):
     Args:
         db: database instance
         table: the table name
-        selections: list of json paths
-        where: a where expression (see below)
-        sort_by: list of json_paths to sort by
+        selections: list of jsonpaths
+        where: a where expression
+        sort_by: list of jsonpaths to sort by
     """
     selections = selections if selections else ["$"]
     if sort_by is None:
@@ -102,7 +103,7 @@ def select_where(db, table, where):
 
 def sort_function(item, sort_by):
     """
-    Given a list of json paths, return a
+    Given a list of jsonpaths, return a
     tuple of associated values for sorting.
     """
     results = get_items(item, sort_by)
@@ -111,7 +112,7 @@ def sort_function(item, sort_by):
 
 def get_items(row, selections):
     """
-    Given a list of json paths, get the
+    Given a list of jsonpaths, get the
     items from the row.
     """
     results = {}
@@ -121,6 +122,7 @@ def get_items(row, selections):
         else:
             match = match_json_path(json_path, row)
             if match:
+                # Remove jsonpath syntax:
                 results[json_path[2:]] = match[0].value
     return results
 
@@ -136,7 +138,13 @@ def match_json_path(json_path, row):
     match = jsonpath_expr.find(row)
     return match
 
+
 def eval_expr(expr, data):
+    """
+    Evaluate the expr. If the expr is a jsonpath,
+    then get the matching value, else all
+    other values are assumed to be self-evaluating.
+    """
     if isinstance(expr, str) and expr.startswith("$"):
         match = match_json_path(expr, data)
         if match:
@@ -149,8 +157,8 @@ def eval_expr(expr, data):
 
 def match_where(data, where):
     """
-    Assumes all comparisons are using JSONpath
-    ("gender", "=", value)
+    Assumes all comparisons are using jsonpath
+    ("$.gender", "=", value)
     """
     if len(where) == 3:
         lhs = eval_expr(where[0], data)
@@ -183,12 +191,19 @@ def compare_where(lhs, op, rhs):
         return lhs != rhs
     elif op == "<":
         return lhs < rhs
+    elif op == "<=":
+        return lhs <= rhs
     elif op == ">":
         return lhs > rhs
+    elif op == ">=":
+        return lhs >= rhs
     elif op == "in":
         return lhs in rhs
     elif op == "not in":
         return lhs not in rhs
+    elif op == "like":
+        pattern = rhs.replace("%", ".*").replace("_", ".")
+        return re.match("^" + pattern + "$", lhs) is not None
     else:
         raise Exception("Operator %r is not supported" % op)
 

From 54de88eaf9e3c4008382d5c4c8a12fb80f93bedb Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 10 Nov 2024 21:01:23 -0500
Subject: [PATCH 03/46] Implementation in SQL

---
 gramps/plugins/db/dbapi/dbapi.py | 44 ++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index a2a90394c8..b5682a93df 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -31,6 +31,7 @@
 import logging
 import json
 import time
+import numbers
 
 from gramps.gen.const import GRAMPS_LOCALE as glocale
 
@@ -1182,3 +1183,46 @@ def _sql_cast_list(self, values):
         in the appropriate type.
         """
         return [v if not isinstance(v, bool) else int(v) for v in values]
+
+    def select(self, table, selections=None, where=None, sort_by=None):
+        def convert_expr(expr):
+            if expr.startswith("$."):
+                return 'json_data->>"%s"' % expr
+            elif isinstance(expr, numbers.Number):
+                return expr
+            elif isinstance(expr, str):
+                return "'%s'" % expr
+            elif expr is None:
+                return "null"
+            elif expr is True:
+                return 1
+            elif expr is False:
+                return 0
+            else:
+                return expr
+
+        def convert_where_expr(where):
+            if len(where) == 3:
+                lhs = convert_expr(where[0])
+                op = where[1]
+                rhs = convert_expr(where[2])
+                return "%s %s %s" % (lhs, op, rhs)
+            else:
+                cond = where[0]
+                expressions = [convert_where_expr(expr) for expr in where[1]]
+                return (" " + cond + " ").join(expressions)
+
+        selections = selections if selections else []
+        sort_by = sort_by if sort_by else []
+        select_clause = ", ".join([('json_data->>"%s"' % item) for item in selections])
+        where_clause = convert_where_expr(where) if where else ""
+        sort_by_clause = ", ".join([('json_data->>"%s"' % item) for item in sort_by])
+
+        self.dbapi.execute(
+            f"select {select_clause} from {table} WHERE {where_clause} ORDER BY {sort_by_clause};"
+        )
+        for row in self.dbapi.fetchall():
+            if "$" in selections:
+                yield row[0]
+            else:
+                yield dict(zip([s[2:] for s in selections], row))

From e1296654d6964f0dcc61f5d6a2c38cedbe809d01 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 11 Nov 2024 12:14:04 -0500
Subject: [PATCH 04/46] Added page, page_size; refactor for better SQL
 variations

---
 gramps/gen/db/generic.py         |  25 ++++++-
 gramps/gen/db/select_utils.py    |  22 +++++-
 gramps/plugins/db/dbapi/dbapi.py | 123 +++++++++++++++++++++++--------
 3 files changed, 132 insertions(+), 38 deletions(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index 25fd9a1337..b47f01a5e1 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -2738,7 +2738,8 @@ def set_serializer(self, serializer_name):
         elif serializer_name == "json":
             self.serializer = JSONSerializer
 
-    def select(self, table, selections=None, where=None, sort_by=None):
+    def select(self, table, selections=None, where=None, sort_by=None,
+               page=0, page_size=25):
         """
         Generic implementation of select, with where and sort_by clauses.
 
@@ -2748,8 +2749,10 @@ def select(self, table, selections=None, where=None, sort_by=None):
         :type selections: List[str] or tuple(str)
         :param where: A single where-expression (see below)
         :type where: tuple or list
-        :param sort_by: A list of expressions to sort on
-        :type where: tuple or list
+        :param page: The page number to return (zero-based)
+        :type page: int
+        :param page_size: The size of a page in rows; None means ignore
+        :type page: int or None
         :returns: Returns selected items from select rows, from iterator
         :rtype: dict
 
@@ -2764,7 +2767,21 @@ def select(self, table, selections=None, where=None, sort_by=None):
             sort_by=["$.gramps_id"],
         )
         ```
+        Notes:
+
+        Although the Python jsonpath_ng library may support more
+        variations than SQL (or other implementations) you should
+        not use them to ensure that your code will run with all
+        backends.
+
+        The where expressions only support the following operators:
+            "=", "!=", "<", "<=", ">", ">=", "in", "not in", "like"
+
+        The `like` operator supports "%" (zero or more characters)
+        and "_" (one character) regular expression matches.
         """
         from gramps.gen.db.select_utils import select
 
-        yield from select(self, table, selections, where, sort_by)
+        yield from select(
+            self, table, selections, where, sort_by, page, page_size
+        )
diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
index 93b165bc2a..70b091225c 100644
--- a/gramps/gen/db/select_utils.py
+++ b/gramps/gen/db/select_utils.py
@@ -28,7 +28,7 @@
 PARSE_CACHE = {}
 
 
-def select(db, table, selections, where, sort_by):
+def select(db, table, selections, where, sort_by, page, page_size):
     """
     Top-level function for select functions.
 
@@ -40,11 +40,27 @@ def select(db, table, selections, where, sort_by):
         sort_by: list of jsonpaths to sort by
     """
     selections = selections if selections else ["$"]
+    if page_size is None:
+        limit = float("+inf")
+        offset = 0
+    else:
+        offset = page * page_size
+        limit = page_size
+
     if sort_by is None:
-        yield from select_items(db, table, selections, where)
+        for count, row in enumerate(select_items(db, table, selections, where)):
+            if count < offset:
+                continue
+            if count > (offset + limit):
+                break
+            yield row
     else:
         results = list(select_items(db, table, ["$"], where))
-        for row in sorted(results, key=lambda item: sort_function(item, sort_by)):
+        for count, row in enumerate(sorted(results, key=lambda item: sort_function(item, sort_by))):
+            if count < offset:
+                continue
+            if count > (offset + limit):
+                break
             yield get_items(row, selections)
 
 
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index b5682a93df..3c2fa4f13f 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1184,42 +1184,103 @@ def _sql_cast_list(self, values):
         """
         return [v if not isinstance(v, bool) else int(v) for v in values]
 
-    def select(self, table, selections=None, where=None, sort_by=None):
-        def convert_expr(expr):
-            if expr.startswith("$."):
-                return 'json_data->>"%s"' % expr
-            elif isinstance(expr, numbers.Number):
-                return expr
-            elif isinstance(expr, str):
-                return "'%s'" % expr
-            elif expr is None:
-                return "null"
-            elif expr is True:
-                return 1
-            elif expr is False:
-                return 0
-            else:
-                return expr
-
-        def convert_where_expr(where):
-            if len(where) == 3:
-                lhs = convert_expr(where[0])
-                op = where[1]
-                rhs = convert_expr(where[2])
-                return "%s %s %s" % (lhs, op, rhs)
+    def _convert_expr_to_sql(self, expr):
+        """
+        This method can be overloaded for different SQL
+        jsonpath syntaxes.
+        """
+        if expr.startswith("$"):
+            return 'json_data->>"%s"' % expr
+        elif isinstance(expr, numbers.Number):
+            return expr
+        elif isinstance(expr, str):
+            return "'%s'" % expr
+        elif expr is None:
+            return "null"
+        elif expr is True:
+            return 1
+        elif expr is False:
+            return 0
+        else:
+            return expr
+
+    def _convert_where_expr_to_sql(self, where):
+        """
+        This method can be overloaded for different SQL
+        syntaxes.
+        """
+        if len(where) == 3:
+            lhs = self._convert_expr_to_sql(where[0])
+            op = where[1]
+            rhs = self._convert_expr_to_sql(where[2])
+            return "(%s %s %s)" % (lhs, op, rhs)
+        else:
+            cond = where[0]
+            expressions = [self._convert_where_expr_to_sql(expr) for expr in where[1]]
+            expressions_str = (" " + cond + " ").join(expressions)
+            if len(expressions) > 1:
+                return "(%s)" % expressions_str
             else:
-                cond = where[0]
-                expressions = [convert_where_expr(expr) for expr in where[1]]
-                return (" " + cond + " ").join(expressions)
+                return expressions_str
+
+    def select(
+            self, table, selections=None, where=None, sort_by=None,
+            page=0, page_size=25,
+    ):
+        """This is a DB-API implementation of the DbGeneric.select()
+        method.
+
+        :param table: Name of table
+        :type table: str
+        :param selections: List of json-paths
+        :type selections: List[str] or tuple(str)
+        :param where: A single where-expression (see below)
+        :type where: tuple or list
+        :param sort_by: A list of expressions to sort on
+        :type where: tuple or list
+        :param page: The page number to return (zero-based)
+        :type page: int
+        :param page_size: The size of a page in rows; None means ignore
+        :type page: int or None
+        :returns: Returns selected items from select rows, from iterator
+        :rtype: dict
+
+        Examples:
+
+        Get the gender and surname of all males, sort by gramps_id:
+        ```
+        db.select(
+            "person",
+            ["$.gender", "$.primary_name.surname_list[0].surname"],
+            where=["$.gender", "=", Person.MALE],
+            sort_by=["$.gramps_id"],
+        )
+        ```
+        Notes:
+
+        Although the SQL engine may support more variations than
+        Python (or other implementations) you should not use them to
+        ensure that your code will run with all backends.
+
+        The where expressions only support the following operators:
+            "=", "!=", "<", "<=", ">", ">=", "in", "not in", "like"
 
-        selections = selections if selections else []
+        The `like` operator supports "%" (zero or more characters)
+        and "_" (one character) regular expression matches.
+
+        """
+        selections = selections if selections else ["$"]
+        select_clause = ", ".join([self._convert_expr_to_sql(item) for item in selections])
+        where_clause = self._convert_where_expr_to_sql(where) if where else ""
         sort_by = sort_by if sort_by else []
-        select_clause = ", ".join([('json_data->>"%s"' % item) for item in selections])
-        where_clause = convert_where_expr(where) if where else ""
-        sort_by_clause = ", ".join([('json_data->>"%s"' % item) for item in sort_by])
+        sort_by_clause = ", ".join([self._convert_expr_to_sql(item) for item in sort_by])
+        if page_size is None:
+            offset_limit_clause = ""
+        else:
+            offset_limit_clause = f"LIMIT {page_size} OFFSET {page * page_size}"
 
         self.dbapi.execute(
-            f"select {select_clause} from {table} WHERE {where_clause} ORDER BY {sort_by_clause};"
+            f"select {select_clause} from {table} WHERE {where_clause} ORDER BY {sort_by_clause} {offset_limit_clause};"
         )
         for row in self.dbapi.fetchall():
             if "$" in selections:

From 87d2c910ccaeedc82328d4ab609c76b7db3cecc9 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 11 Nov 2024 12:15:00 -0500
Subject: [PATCH 05/46] Linting

---
 gramps/gen/db/generic.py         |  9 ++++-----
 gramps/gen/db/select_utils.py    |  4 +++-
 gramps/plugins/db/dbapi/dbapi.py | 17 +++++++++++++----
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index b47f01a5e1..57631dd346 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -2738,8 +2738,9 @@ def set_serializer(self, serializer_name):
         elif serializer_name == "json":
             self.serializer = JSONSerializer
 
-    def select(self, table, selections=None, where=None, sort_by=None,
-               page=0, page_size=25):
+    def select(
+        self, table, selections=None, where=None, sort_by=None, page=0, page_size=25
+    ):
         """
         Generic implementation of select, with where and sort_by clauses.
 
@@ -2782,6 +2783,4 @@ def select(self, table, selections=None, where=None, sort_by=None,
         """
         from gramps.gen.db.select_utils import select
 
-        yield from select(
-            self, table, selections, where, sort_by, page, page_size
-        )
+        yield from select(self, table, selections, where, sort_by, page, page_size)
diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
index 70b091225c..949eef873a 100644
--- a/gramps/gen/db/select_utils.py
+++ b/gramps/gen/db/select_utils.py
@@ -56,7 +56,9 @@ def select(db, table, selections, where, sort_by, page, page_size):
             yield row
     else:
         results = list(select_items(db, table, ["$"], where))
-        for count, row in enumerate(sorted(results, key=lambda item: sort_function(item, sort_by))):
+        for count, row in enumerate(
+            sorted(results, key=lambda item: sort_function(item, sort_by))
+        ):
             if count < offset:
                 continue
             if count > (offset + limit):
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 3c2fa4f13f..4efdfb16fb 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1224,8 +1224,13 @@ def _convert_where_expr_to_sql(self, where):
                 return expressions_str
 
     def select(
-            self, table, selections=None, where=None, sort_by=None,
-            page=0, page_size=25,
+        self,
+        table,
+        selections=None,
+        where=None,
+        sort_by=None,
+        page=0,
+        page_size=25,
     ):
         """This is a DB-API implementation of the DbGeneric.select()
         method.
@@ -1270,10 +1275,14 @@ def select(
 
         """
         selections = selections if selections else ["$"]
-        select_clause = ", ".join([self._convert_expr_to_sql(item) for item in selections])
+        select_clause = ", ".join(
+            [self._convert_expr_to_sql(item) for item in selections]
+        )
         where_clause = self._convert_where_expr_to_sql(where) if where else ""
         sort_by = sort_by if sort_by else []
-        sort_by_clause = ", ".join([self._convert_expr_to_sql(item) for item in sort_by])
+        sort_by_clause = ", ".join(
+            [self._convert_expr_to_sql(item) for item in sort_by]
+        )
         if page_size is None:
             offset_limit_clause = ""
         else:

From 4dbed251327612f743c9b9839683062fb862b6c1 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 11 Nov 2024 14:38:06 -0500
Subject: [PATCH 06/46] Fixed a bug in indexable where clause; change page_size
 default

---
 gramps/gen/db/generic.py         |  2 +-
 gramps/gen/db/select_utils.py    | 25 +++++++++++++++++++------
 gramps/plugins/db/dbapi/dbapi.py | 19 ++++++++++++-------
 3 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index 57631dd346..b1ad404842 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -2739,7 +2739,7 @@ def set_serializer(self, serializer_name):
             self.serializer = JSONSerializer
 
     def select(
-        self, table, selections=None, where=None, sort_by=None, page=0, page_size=25
+        self, table, selections=None, where=None, sort_by=None, page=0, page_size=None
     ):
         """
         Generic implementation of select, with where and sort_by clauses.
diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
index 949eef873a..6b6401bc96 100644
--- a/gramps/gen/db/select_utils.py
+++ b/gramps/gen/db/select_utils.py
@@ -88,9 +88,19 @@ def select_where(db, table, where):
     handle and gramps_id lookup methods.
     """
     if where:
+        # Check for a simple optimization:
+        # is where composed of indexable expressions?
         indexed_expressions = []
-        get_indexed_fields(where, indexed_expressions)
-        if indexed_expressions:
+        non_indexed_expressions = []
+        connectors = []
+        get_indexed_fields(
+            where, indexed_expressions, non_indexed_expressions, connectors
+        )
+        # Has to have some indexed, and either no non-indexed
+        # or only non-indexed combined with "and"
+        if indexed_expressions and (
+            (not non_indexed_expressions) or ("or" not in connectors)
+        ):
             handles_returned = []
             for indexed in indexed_expressions:
                 index_field, compare, index_value = indexed
@@ -226,10 +236,10 @@ def compare_where(lhs, op, rhs):
         raise Exception("Operator %r is not supported" % op)
 
 
-def get_indexed_fields(where, indexes):
+def get_indexed_fields(where, indexes, non_indexes, connectors):
     """
-    Recursive function that accumpulates matches
-    in the mutable `indexes` list.
+    Recursive function that accumpulates data
+    in the mutable lists.
 
     A match is an expression that compares "$.handle"
     or "$.gramps_id" equal to a value.
@@ -251,8 +261,11 @@ def get_indexed_fields(where, indexes):
         # A comparison expression
         if where[0] in ["$.gramps_id", "$.handle"] and where[1] == "=":
             indexes += [where]
+        else:
+            non_indexes += [where]
     else:
         # Must be ("and", exprs) or ("or", exprs)
         # Recurse on all expressions
+        connectors += [where[0]]
         for expr in where[1]:
-            get_indexed_fields(expr, indexes)
+            get_indexed_fields(expr, indexes, non_indexes, connectors)
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 4efdfb16fb..46d12e894a 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1189,12 +1189,13 @@ def _convert_expr_to_sql(self, expr):
         This method can be overloaded for different SQL
         jsonpath syntaxes.
         """
-        if expr.startswith("$"):
-            return 'json_data->>"%s"' % expr
-        elif isinstance(expr, numbers.Number):
+        if isinstance(expr, numbers.Number):
             return expr
         elif isinstance(expr, str):
-            return "'%s'" % expr
+            if expr.startswith("$"):
+                return 'json_data->>"%s"' % expr
+            else:
+                return "'%s'" % expr
         elif expr is None:
             return "null"
         elif expr is True:
@@ -1230,7 +1231,7 @@ def select(
         where=None,
         sort_by=None,
         page=0,
-        page_size=25,
+        page_size=None,
     ):
         """This is a DB-API implementation of the DbGeneric.select()
         method.
@@ -1279,17 +1280,21 @@ def select(
             [self._convert_expr_to_sql(item) for item in selections]
         )
         where_clause = self._convert_where_expr_to_sql(where) if where else ""
+        if where_clause:
+            where_clause = " WHERE" + where_clause
         sort_by = sort_by if sort_by else []
         sort_by_clause = ", ".join(
             [self._convert_expr_to_sql(item) for item in sort_by]
         )
+        if sort_by_clause:
+            sort_by_clause = " ORDER BY " + sort_by_clause
         if page_size is None:
             offset_limit_clause = ""
         else:
-            offset_limit_clause = f"LIMIT {page_size} OFFSET {page * page_size}"
+            offset_limit_clause = f" LIMIT {page_size} OFFSET {page * page_size}"
 
         self.dbapi.execute(
-            f"select {select_clause} from {table} WHERE {where_clause} ORDER BY {sort_by_clause} {offset_limit_clause};"
+            f"select {select_clause} from {table}{where_clause}{sort_by_clause}{offset_limit_clause};"
         )
         for row in self.dbapi.fetchall():
             if "$" in selections:

From e21bcc82ddfd3f3869d57e7b11fa409d70042a80 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 11 Nov 2024 21:22:42 -0500
Subject: [PATCH 07/46] ((lhs, 'and', rhs), 'or' (lhs, 'and', rhs))

---
 gramps/gen/db/select_utils.py    | 104 ++++++++++++++++++++++---------
 gramps/plugins/db/dbapi/dbapi.py |  15 ++---
 2 files changed, 80 insertions(+), 39 deletions(-)

diff --git a/gramps/gen/db/select_utils.py b/gramps/gen/db/select_utils.py
index 6b6401bc96..4e8a413986 100644
--- a/gramps/gen/db/select_utils.py
+++ b/gramps/gen/db/select_utils.py
@@ -26,18 +26,50 @@
 import jsonpath_ng
 
 PARSE_CACHE = {}
+OPERATORS = ("=", "!=", "<", "<=", ">", ">=", "in", "not in", "like")
 
 
 def select(db, table, selections, where, sort_by, page, page_size):
-    """
-    Top-level function for select functions.
-
-    Args:
-        db: database instance
-        table: the table name
-        selections: list of jsonpaths
-        where: a where expression
-        sort_by: list of jsonpaths to sort by
+    """This is a DB-API implementation of the DbGeneric.select()
+    method.
+
+    :param table: Name of table
+    :type table: str
+    :param selections: List of json-paths
+    :type selections: List[str] or tuple(str)
+    :param where: A single where-expression (see below)
+    :type where: tuple or list
+    :param sort_by: A list of expressions to sort on
+    :type where: tuple or list
+    :param page: The page number to return (zero-based)
+    :type page: int
+    :param page_size: The size of a page in rows; None means ignore
+    :type page: int or None
+    :returns: Returns selected items from select rows, from iterator
+    :rtype: dict
+
+    Examples:
+
+    Get the gender and surname of all males, sort by gramps_id:
+    ```
+    db.select(
+        "person",
+        ["$.gender", "$.primary_name.surname_list[0].surname"],
+        where=["$.gender", "=", Person.MALE],
+        sort_by=["$.gramps_id"],
+    )
+    ```
+    Notes:
+
+    Although the SQL engine may support more variations than
+    Python (or other implementations) you should not use them to
+    ensure that your code will run with all backends.
+
+    The where expressions only support the following operators:
+        "=", "!=", "<", "<=", ">", ">=", "in", "not in", "like"
+
+    The `like` operator supports "%" (zero or more characters)
+    and "_" (one character) regular expression matches.
     """
     selections = selections if selections else ["$"]
     if page_size is None:
@@ -104,6 +136,12 @@ def select_where(db, table, where):
             handles_returned = []
             for indexed in indexed_expressions:
                 index_field, compare, index_value = indexed
+                if isinstance(index_value, str) and index_value.startswith("$"):
+                    index_value, compare, index_field = (
+                        index_field,
+                        compare,
+                        index_value,
+                    )
                 if index_field == "$.gramps_id":
                     func_name = "raw_id_func"
                 elif index_field == "$.handle":
@@ -189,22 +227,25 @@ def match_where(data, where):
     ("$.gender", "=", value)
     """
     if len(where) == 3:
-        lhs = eval_expr(where[0], data)
-        op = where[1].lower()
-        rhs = eval_expr(where[2], data)
-        return compare_where(lhs, op, rhs)
-    elif where[0].lower() == "and":
-        for expr in where[1]:
-            result = match_where(data, expr)
-            if not result:
-                return False
-        return True
-    elif where[0].lower() == "or":
-        for expr in where[1]:
-            result = match_where(data, expr)
-            if result:
-                return True
-        return False
+        if where[1].lower() == "and":
+            for expr in [where[0], where[2]]:
+                result = match_where(data, expr)
+                if not result:
+                    return False
+            return True
+        elif where[1].lower() == "or":
+            for expr in [where[0], where[2]]:
+                result = match_where(data, expr)
+                if result:
+                    return True
+            return False
+        elif where[1].lower() in OPERATORS:
+            lhs = eval_expr(where[0], data)
+            op = where[1].lower()
+            rhs = eval_expr(where[2], data)
+            return compare_where(lhs, op, rhs)
+        else:
+            raise Exception("Operator %r is not supported" % where[1])
     else:
         raise Exception("Malformed where expression: %r" % where)
 
@@ -257,15 +298,18 @@ def get_indexed_fields(where, indexes, non_indexes, connectors):
     ("or", (...))
     ```
     """
-    if len(where) == 3:
+    if where[1] in OPERATORS:
         # A comparison expression
-        if where[0] in ["$.gramps_id", "$.handle"] and where[1] == "=":
+        if (
+            (where[0] in ["$.gramps_id", "$.handle"])
+            or (where[2] in ["$.gramps_id", "$.handle"])
+        ) and where[1] == "=":
             indexes += [where]
         else:
             non_indexes += [where]
     else:
-        # Must be ("and", exprs) or ("or", exprs)
+        # Must be (lhs, "and", rhs) or (lhs, "or", rhs)
         # Recurse on all expressions
-        connectors += [where[0]]
-        for expr in where[1]:
+        connectors += [where[1]]
+        for expr in [where[0], where[2]]:
             get_indexed_fields(expr, indexes, non_indexes, connectors)
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 46d12e894a..2140432924 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1210,19 +1210,16 @@ def _convert_where_expr_to_sql(self, where):
         This method can be overloaded for different SQL
         syntaxes.
         """
-        if len(where) == 3:
+        if where[1].lower() in ["and", "or"]:
+            lhs = self._convert_where_expr_to_sql(where[0])
+            cond = where[1]
+            rhs = self._convert_where_expr_to_sql(where[2])
+            return "(%s %s %s)" % (lhs, cond, rhs)
+        else:
             lhs = self._convert_expr_to_sql(where[0])
             op = where[1]
             rhs = self._convert_expr_to_sql(where[2])
             return "(%s %s %s)" % (lhs, op, rhs)
-        else:
-            cond = where[0]
-            expressions = [self._convert_where_expr_to_sql(expr) for expr in where[1]]
-            expressions_str = (" " + cond + " ").join(expressions)
-            if len(expressions) > 1:
-                return "(%s)" % expressions_str
-            else:
-                return expressions_str
 
     def select(
         self,

From 0de54ea5082caf895a86fa9dfc832a2a7ddcc777 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Thu, 14 Nov 2024 05:12:09 -0500
Subject: [PATCH 08/46] Properly replace values with ?

---
 gramps/plugins/db/dbapi/dbapi.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 2140432924..3a0e1acb97 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1184,7 +1184,7 @@ def _sql_cast_list(self, values):
         """
         return [v if not isinstance(v, bool) else int(v) for v in values]
 
-    def _convert_expr_to_sql(self, expr):
+    def _convert_expr_to_sql(self, expr, values):
         """
         This method can be overloaded for different SQL
         jsonpath syntaxes.
@@ -1195,7 +1195,8 @@ def _convert_expr_to_sql(self, expr):
             if expr.startswith("$"):
                 return 'json_data->>"%s"' % expr
             else:
-                return "'%s'" % expr
+                values.append(expr)
+                return "?"
         elif expr is None:
             return "null"
         elif expr is True:
@@ -1203,22 +1204,23 @@ def _convert_expr_to_sql(self, expr):
         elif expr is False:
             return 0
         else:
-            return expr
+            values.append(expr)
+            return "?"
 
-    def _convert_where_expr_to_sql(self, where):
+    def _convert_where_expr_to_sql(self, where, values):
         """
         This method can be overloaded for different SQL
         syntaxes.
         """
         if where[1].lower() in ["and", "or"]:
-            lhs = self._convert_where_expr_to_sql(where[0])
+            lhs = self._convert_where_expr_to_sql(where[0], values)
             cond = where[1]
-            rhs = self._convert_where_expr_to_sql(where[2])
+            rhs = self._convert_where_expr_to_sql(where[2], values)
             return "(%s %s %s)" % (lhs, cond, rhs)
         else:
-            lhs = self._convert_expr_to_sql(where[0])
+            lhs = self._convert_expr_to_sql(where[0], values)
             op = where[1]
-            rhs = self._convert_expr_to_sql(where[2])
+            rhs = self._convert_expr_to_sql(where[2], values)
             return "(%s %s %s)" % (lhs, op, rhs)
 
     def select(
@@ -1274,14 +1276,15 @@ def select(
         """
         selections = selections if selections else ["$"]
         select_clause = ", ".join(
-            [self._convert_expr_to_sql(item) for item in selections]
+            [self._convert_expr_to_sql(item, []) for item in selections]
         )
-        where_clause = self._convert_where_expr_to_sql(where) if where else ""
+        values = []
+        where_clause = self._convert_where_expr_to_sql(where, values) if where else ""
         if where_clause:
-            where_clause = " WHERE" + where_clause
+            where_clause = " WHERE " + where_clause
         sort_by = sort_by if sort_by else []
         sort_by_clause = ", ".join(
-            [self._convert_expr_to_sql(item) for item in sort_by]
+            [self._convert_expr_to_sql(item, []) for item in sort_by]
         )
         if sort_by_clause:
             sort_by_clause = " ORDER BY " + sort_by_clause
@@ -1291,7 +1294,8 @@ def select(
             offset_limit_clause = f" LIMIT {page_size} OFFSET {page * page_size}"
 
         self.dbapi.execute(
-            f"select {select_clause} from {table}{where_clause}{sort_by_clause}{offset_limit_clause};"
+            f"select {select_clause} from {table}{where_clause}{sort_by_clause}{offset_limit_clause};",
+            values
         )
         for row in self.dbapi.fetchall():
             if "$" in selections:

From 8d14dd3d94bba89ec6bec3c80a3c5337844c084b Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 14:08:02 -0500
Subject: [PATCH 09/46] Framework for filter refactor

---
 gramps/gen/filters/_genericfilter.py          | 104 ++++++++----------
 gramps/gen/filters/rules/_hastagbase.py       |  13 ++-
 .../gen/filters/rules/_matchesfilterbase.py   |   5 +-
 gramps/gen/filters/rules/person/_hastag.py    |   1 +
 .../gen/filters/rules/person/_isancestorof.py |   4 +-
 gramps/gui/views/treemodels/flatbasemodel.py  |   4 +-
 6 files changed, 62 insertions(+), 69 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 02dcfd2b82..3385a89cfb 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -71,11 +71,11 @@ def __init__(self, source=None):
             self.logical_op = "and"
             self.invert = False
 
-    def match(self, handle, db):
+    def match(self, data, db):
         """
         Return True or False depending on whether the handle matches the filter.
         """
-        if self.apply(db, [handle]):
+        if self.apply_to_one(db, data):
             return True
         else:
             return False
@@ -139,101 +139,77 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_people()
 
-    def check_func(self, db, id_list, task, user=None, tupleind=None, tree=False):
+    def check_func(self, db, id_list, logical_test, user=None, tupleind=None, tree=False):
         final_list = []
         if user:
             user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
-        if id_list is None:
-            with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
-                for handle, data in cursor:
-                    person = from_dict(data)
-                    if user:
-                        user.step_progress()
-                    if task(db, person) != self.invert:
-                        final_list.append(handle)
-        else:
-            for data in id_list:
-                if tupleind is None:
-                    handle = data
-                else:
-                    handle = data[tupleind]
-                person = self.find_from_handle(db, handle)
-                if user:
-                    user.step_progress()
-                if task(db, person) != self.invert:
-                    final_list.append(data)
-        if user:
-            user.end_progress()
-        return final_list
 
-    def check_and(self, db, id_list, user=None, tupleind=None, tree=False):
-        final_list = []
-        flist = self.flist
-        if user:
-            user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
         if id_list is None:
             with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
                 for handle, data in cursor:
-                    person = from_dict(data)
                     if user:
                         user.step_progress()
-                    val = all(rule.apply(db, person) for rule in flist)
-                    if val != self.invert:
+
+                    if logical_test(db, data) != self.invert:
                         final_list.append(handle)
+
         else:
             for data in id_list:
                 if tupleind is None:
                     handle = data
                 else:
                     handle = data[tupleind]
-                person = self.find_from_handle(db, handle)
+
+                json_data = db.get_raw_person_data(handle)
+
                 if user:
                     user.step_progress()
-                val = all(rule.apply(db, person) for rule in flist if person)
-                if val != self.invert:
+
+                if logical_test(db, json_data) != self.invert:
                     final_list.append(data)
+
         if user:
             user.end_progress()
+
         return final_list
 
     def check_or(self, db, id_list, user=None, tupleind=None, tree=False):
         return self.check_func(db, id_list, self.or_test, user, tupleind, tree=False)
 
+    def check_and(self, db, id_list, user=None, tupleind=None, tree=False):
+        return self.check_func(db, id_list, self.and_test, user, tupleind, tree=False)
+
     def check_one(self, db, id_list, user=None, tupleind=None, tree=False):
         return self.check_func(db, id_list, self.one_test, user, tupleind, tree=False)
 
-    def check_xor(self, db, id_list, user=None, tupleind=None, tree=False):
-        return self.check_func(db, id_list, self.xor_test, user, tupleind, tree=False)
+    def and_test(self, db, data):
+        return all(rule.apply_to_one(db, data) for rule in self.flist)
 
-    def xor_test(self, db, person):
-        test = False
-        for rule in self.flist:
-            test = test ^ rule.apply(db, person)
-        return test
-
-    def one_test(self, db, person):
+    def one_test(self, db, data):
         found_one = False
         for rule in self.flist:
-            if rule.apply(db, person):
+            if rule.apply_to_one(db, data):
                 if found_one:
                     return False  # There can be only one!
                 found_one = True
         return found_one
 
-    def or_test(self, db, person):
-        return any(rule.apply(db, person) for rule in self.flist)
+    def or_test(self, db, data):
+        return any(rule.apply_to_one(db, data) for rule in self.flist)
 
-    def get_check_func(self):
-        try:
-            m = getattr(self, "check_" + self.logical_op)
-        except AttributeError:
-            m = self.check_and
-        return m
+    def apply_to_one(self, db, data):
+        if self.logical_op == "and":
+            res = self.and_test(db, data)
+        elif self.logical_op == "or":
+            res = self.or_test(db, data)
+        elif self.logical_op == "one":
+            res = self.one_test(db, data)
+        else:
+            raise Exception("invalid operator: %r" % self.logical_op)
+        return res != self.invert
 
-    def check(self, db, handle):
-        return self.get_check_func()(db, [handle])
 
-    def apply(self, db, id_list=None, tupleind=None, user=None, tree=False):
+    def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
         """
         Apply the filter using db.
         If id_list given, the handles in id_list are used. If not given
@@ -250,12 +226,22 @@ def apply(self, db, id_list=None, tupleind=None, user=None, tree=False):
                 if id_list not given, all items in the database that
                 match the filter are returned as a list of handles
         """
-        m = self.get_check_func()
+        
         for rule in self.flist:
             rule.requestprepare(db, user)
-        res = m(db, id_list, user, tupleind, tree)
+
+        if self.logical_op == "and":
+            res = self.check_and(db, id_list, user, tupleind, tree)
+        elif self.logical_op == "or":
+            res = self.check_or(db, id_list, user, tupleind, tree)
+        elif self.logical_op == "one":
+            res = self.check_one(db, id_list, user, tupleind, tree)
+        else:
+            raise Exception("invalid operator: %r" % self.logical_op)
+
         for rule in self.flist:
             rule.requestreset()
+
         return res
 
 
diff --git a/gramps/gen/filters/rules/_hastagbase.py b/gramps/gen/filters/rules/_hastagbase.py
index 9d70877612..73a1c5effe 100644
--- a/gramps/gen/filters/rules/_hastagbase.py
+++ b/gramps/gen/filters/rules/_hastagbase.py
@@ -61,11 +61,16 @@ def prepare(self, db, user):
         tag = db.get_tag_from_name(self.list[0])
         if tag is not None:
             self.tag_handle = tag.get_handle()
+            results = db.select(self.table, ["$.handle"], ("$.tag_list", "LIKE", f'%"{self.tag_handle}"%'))
+            self.map = set([row["handle"] for row in list(results)])
+        else:
+            self.map = set()
 
-    def apply(self, db, obj):
+    def get_rules_with_maps(self):
+        return [self]
+            
+    def apply_to_one(self, db, data):
         """
         Apply the rule.  Return True for a match.
         """
-        if self.tag_handle is None:
-            return False
-        return self.tag_handle in obj.get_tag_list()
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/_matchesfilterbase.py b/gramps/gen/filters/rules/_matchesfilterbase.py
index 208b876e75..df979a47fb 100644
--- a/gramps/gen/filters/rules/_matchesfilterbase.py
+++ b/gramps/gen/filters/rules/_matchesfilterbase.py
@@ -85,12 +85,13 @@ def reset(self):
                 for rule in filt.flist:
                     rule.requestreset()
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         if gramps.gen.filters.CustomFilters:
             filters = gramps.gen.filters.CustomFilters.get_filters_dict(self.namespace)
             if self.list[0] in filters:
                 filt = filters[self.list[0]]
-                return filt.check(db, obj.handle)
+                # FIXME: Is this correct?
+                return filt.apply_to_one(db, data)
         return False
 
     def find_filter(self):
diff --git a/gramps/gen/filters/rules/person/_hastag.py b/gramps/gen/filters/rules/person/_hastag.py
index 2378b9a9f0..f2c2280340 100644
--- a/gramps/gen/filters/rules/person/_hastag.py
+++ b/gramps/gen/filters/rules/person/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("People with the <tag>")
     description = _("Matches people with the particular tag")
+    table = "person"
diff --git a/gramps/gen/filters/rules/person/_isancestorof.py b/gramps/gen/filters/rules/person/_isancestorof.py
index ac2668b4a6..6db18c4665 100644
--- a/gramps/gen/filters/rules/person/_isancestorof.py
+++ b/gramps/gen/filters/rules/person/_isancestorof.py
@@ -65,8 +65,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_ancestor_list(self, db, person, first):
         if not person:
diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py
index 41ab69b7bd..1c7ae9b33a 100644
--- a/gramps/gui/views/treemodels/flatbasemodel.py
+++ b/gramps/gui/views/treemodels/flatbasemodel.py
@@ -627,9 +627,9 @@ def _rebuild_filter(self, ignore=None):
             if self.search:
                 ident = False
                 if ignore is None:
-                    dlist = self.search.apply(cdb, allkeys, tupleind=1, user=self.user)
+                    dlist = self.search.apply_to_all(cdb, allkeys, tupleind=1, user=self.user)
                 else:
-                    dlist = self.search.apply(
+                    dlist = self.search.apply_to_all(
                         cdb, [k for k in allkeys if k[1] != ignore], tupleind=1
                     )
             elif ignore is None:

From c18b7fd13b62ee85f512f1e49b981fef252dc567 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 16:09:17 -0500
Subject: [PATCH 10/46] Walk the filters/rules looking for maps

---
 gramps/gen/filters/_genericfilter.py | 89 ++++++++++++++++++----------
 gramps/gen/filters/rules/_rule.py    |  2 +-
 2 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 3385a89cfb..09d1ddc709 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -41,6 +41,7 @@
 from ..lib.tag import Tag
 from ..lib.serialize import from_dict
 from ..const import GRAMPS_LOCALE as glocale
+from .rules import Rule
 
 _ = glocale.translation.gettext
 
@@ -53,7 +54,7 @@
 class GenericFilter:
     """Filter class that consists of several rules."""
 
-    logical_functions = ["or", "and", "xor", "one"]
+    logical_functions = ["and", "or", "one"]
 
     def __init__(self, source=None):
         if source:
@@ -89,10 +90,7 @@ def set_logical_op(self, val):
         if val in GenericFilter.logical_functions:
             self.logical_op = val
         else:
-            self.logical_op = "and"
-
-    def get_logical_op(self):
-        return self.logical_op
+            raise Exception("invalid operator: %r" % val)
 
     def set_invert(self, val):
         self.invert = bool(val)
@@ -139,18 +137,47 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_people()
 
-    def check_func(self, db, id_list, logical_test, user=None, tupleind=None, tree=False):
+    def walk_filters(self, filter, result):
+        """
+        Walk all of the filters/rules and get
+        rules with maps
+        """
+        rules = []
+        for item in filter.flist:
+            if hasattr(item, "find_filter"):
+                self.walk_filters(item.find_filter(), result)
+            elif hasattr(item, "map"):
+                rules.append(item)
+        if rules:
+            result.append((filter.invert, filter.logical_op, rules))
+
+    def apply_logical_op_to_all(
+        self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
+    ):
         final_list = []
         if user:
             user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
 
+        # Optimiziations
+        print("------------------------")
+        all_maps = []
+        self.walk_filters(self, all_maps)
+        for invert, logical_op, maps in all_maps:
+            print(
+                "handle must be",
+                "not in" if invert else "in",
+                "joined",
+                logical_op,
+                maps,
+            )
+
         if id_list is None:
             with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
                 for handle, data in cursor:
                     if user:
                         user.step_progress()
 
-                    if logical_test(db, data) != self.invert:
+                    if apply_logical_op(db, data, self.flist) != self.invert:
                         final_list.append(handle)
 
         else:
@@ -165,7 +192,7 @@ def check_func(self, db, id_list, logical_test, user=None, tupleind=None, tree=F
                 if user:
                     user.step_progress()
 
-                if logical_test(db, json_data) != self.invert:
+                if apply_logical_op(db, json_data, self.flist) != self.invert:
                     final_list.append(data)
 
         if user:
@@ -173,42 +200,38 @@ def check_func(self, db, id_list, logical_test, user=None, tupleind=None, tree=F
 
         return final_list
 
-    def check_or(self, db, id_list, user=None, tupleind=None, tree=False):
-        return self.check_func(db, id_list, self.or_test, user, tupleind, tree=False)
-
-    def check_and(self, db, id_list, user=None, tupleind=None, tree=False):
-        return self.check_func(db, id_list, self.and_test, user, tupleind, tree=False)
-
-    def check_one(self, db, id_list, user=None, tupleind=None, tree=False):
-        return self.check_func(db, id_list, self.one_test, user, tupleind, tree=False)
-
-    def and_test(self, db, data):
-        return all(rule.apply_to_one(db, data) for rule in self.flist)
+    def and_test(self, db, data, flist):
+        return all(rule.apply_to_one(db, data) for rule in flist)
 
-    def one_test(self, db, data):
+    def one_test(self, db, data, flist):
         found_one = False
-        for rule in self.flist:
+        for rule in flist:
             if rule.apply_to_one(db, data):
                 if found_one:
                     return False  # There can be only one!
                 found_one = True
         return found_one
 
-    def or_test(self, db, data):
-        return any(rule.apply_to_one(db, data) for rule in self.flist)
+    def or_test(self, db, data, flist):
+        return any(rule.apply_to_one(db, data) for rule in flist)
+
+    def get_logical_op(self):
+        return self.logical_op
 
     def apply_to_one(self, db, data):
+        """
+        Filter-level apply rules to single data item.
+        """
         if self.logical_op == "and":
-            res = self.and_test(db, data)
+            res = self.and_test(db, data, self.flist)
         elif self.logical_op == "or":
-            res = self.or_test(db, data)
+            res = self.or_test(db, data, self.flist)
         elif self.logical_op == "one":
-            res = self.one_test(db, data)
+            res = self.one_test(db, data, self.flist)
         else:
             raise Exception("invalid operator: %r" % self.logical_op)
         return res != self.invert
 
-
     def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
         """
         Apply the filter using db.
@@ -226,19 +249,23 @@ def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
                 if id_list not given, all items in the database that
                 match the filter are returned as a list of handles
         """
-        
+
         for rule in self.flist:
             rule.requestprepare(db, user)
 
         if self.logical_op == "and":
-            res = self.check_and(db, id_list, user, tupleind, tree)
+            apply_logical_op = self.and_test
         elif self.logical_op == "or":
-            res = self.check_or(db, id_list, user, tupleind, tree)
+            apply_logical_op = self.or_test
         elif self.logical_op == "one":
-            res = self.check_one(db, id_list, user, tupleind, tree)
+            apply_logical_op = self.one_test
         else:
             raise Exception("invalid operator: %r" % self.logical_op)
 
+        res = self.apply_logical_op_to_all(
+            db, id_list, apply_logical_op, user, tupleind, tree
+        )
+
         for rule in self.flist:
             rule.requestreset()
 
diff --git a/gramps/gen/filters/rules/_rule.py b/gramps/gen/filters/rules/_rule.py
index 0d565dfcc8..544b5c5530 100644
--- a/gramps/gen/filters/rules/_rule.py
+++ b/gramps/gen/filters/rules/_rule.py
@@ -150,7 +150,7 @@ def check(self):
         """Verify the number of rule values versus the number of rule labels."""
         return len(self.list) == len(self.labels)
 
-    def apply(self, dummy_db, dummy_person):
+    def apply_to_one(self, dummy_db, dummy_data):
         """Apply the rule to some database entry; must be overwritten."""
         return True
 

From d3cb85d54b987a2871575c6a02091377ebfe1a21 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 16:27:56 -0500
Subject: [PATCH 11/46] Working optimization

---
 gramps/gen/filters/_genericfilter.py | 50 ++++++++++++++++++----------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 09d1ddc709..73de8dccd0 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -147,7 +147,7 @@ def walk_filters(self, filter, result):
             if hasattr(item, "find_filter"):
                 self.walk_filters(item.find_filter(), result)
             elif hasattr(item, "map"):
-                rules.append(item)
+                rules.append(item.map)
         if rules:
             result.append((filter.invert, filter.logical_op, rules))
 
@@ -158,27 +158,43 @@ def apply_logical_op_to_all(
         if user:
             user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
 
-        # Optimiziations
-        print("------------------------")
-        all_maps = []
-        self.walk_filters(self, all_maps)
-        for invert, logical_op, maps in all_maps:
-            print(
-                "handle must be",
-                "not in" if invert else "in",
-                "joined",
-                logical_op,
-                maps,
-            )
+        def intersection(sets):
+            result = sets[0]
+            for s in sets[1:]:
+                result = result.intersection(s)
+            return result
 
         if id_list is None:
-            with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
-                for handle, data in cursor:
+            # Optimiziation
+            all_maps = []
+            self.walk_filters(self, all_maps)
+            handles = None
+            # Get all positive non-inverted maps
+            for inverted, logical_op, maps in all_maps:
+                if logical_op == "and" and not inverted:
+                    if handles is None:
+                        handles = intersection(maps)
+                    else:
+                        handles = intersection([handles] + maps)
+            if handles:
+                # Use these rather than going through entire database
+                for handle in handles:
+                    json_data = db.get_raw_person_data(handle)
+
                     if user:
                         user.step_progress()
 
-                    if apply_logical_op(db, data, self.flist) != self.invert:
-                        final_list.append(handle)
+                    if apply_logical_op(db, json_data, self.flist) != self.invert:
+                        final_list.append(json_data)
+
+            else:
+                with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
+                    for handle, data in cursor:
+                        if user:
+                            user.step_progress()
+
+                        if apply_logical_op(db, data, self.flist) != self.invert:
+                            final_list.append(handle)
 
         else:
             for data in id_list:

From 5bcd65b3fd74d0e2fe7768f8630985f9dbbc45ed Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 16:38:04 -0500
Subject: [PATCH 12/46] Working optimization; linted

---
 gramps/gen/filters/_genericfilter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 73de8dccd0..fbf5a3467a 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -188,7 +188,9 @@ def intersection(sets):
                         final_list.append(json_data)
 
             else:
-                with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor:
+                with self.get_tree_cursor(db) if tree else self.get_cursor(
+                    db
+                ) as cursor:
                     for handle, data in cursor:
                         if user:
                             user.step_progress()

From e18fad4e57836bf7f059051139858cc585676266 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 16:43:34 -0500
Subject: [PATCH 13/46] Working optimization on on-memory id_list too

---
 gramps/gen/filters/_genericfilter.py | 35 ++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index fbf5a3467a..abb38f2147 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -199,19 +199,34 @@ def intersection(sets):
                             final_list.append(handle)
 
         else:
-            for data in id_list:
-                if tupleind is None:
-                    handle = data
-                else:
-                    handle = data[tupleind]
+            # Optimiziation
+            all_maps = []
+            self.walk_filters(self, all_maps)
+            handles = None
+            # Get all positive non-inverted maps
+            for inverted, logical_op, maps in all_maps:
+                if logical_op == "and" and not inverted:
+                    if handles is None:
+                        handles = intersection(maps)
+                    else:
+                        handles = intersection([handles] + maps)
+            if handles:
+                for data in id_list:
+                    if tupleind is None:
+                        handle = data
+                    else:
+                        handle = data[tupleind]
 
-                json_data = db.get_raw_person_data(handle)
+                    if handle not in handles:
+                        continue
 
-                if user:
-                    user.step_progress()
+                    json_data = db.get_raw_person_data(handle)
 
-                if apply_logical_op(db, json_data, self.flist) != self.invert:
-                    final_list.append(data)
+                    if user:
+                        user.step_progress()
+
+                    if apply_logical_op(db, json_data, self.flist) != self.invert:
+                        final_list.append(data)
 
         if user:
             user.end_progress()

From 296eec13a8c00ad56987f53b359a363afe122dc2 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 16:45:47 -0500
Subject: [PATCH 14/46] Working optimization on on-memory id_list too; revised

---
 gramps/gen/filters/_genericfilter.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index abb38f2147..045bc18226 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -210,23 +210,23 @@ def intersection(sets):
                         handles = intersection(maps)
                     else:
                         handles = intersection([handles] + maps)
-            if handles:
-                for data in id_list:
-                    if tupleind is None:
-                        handle = data
-                    else:
-                        handle = data[tupleind]
 
-                    if handle not in handles:
-                        continue
+            for data in id_list:
+                if tupleind is None:
+                    handle = data
+                else:
+                    handle = data[tupleind]
 
-                    json_data = db.get_raw_person_data(handle)
+                if handles is not None and handle not in handles:
+                    continue
 
-                    if user:
-                        user.step_progress()
+                json_data = db.get_raw_person_data(handle)
 
-                    if apply_logical_op(db, json_data, self.flist) != self.invert:
-                        final_list.append(data)
+                if user:
+                    user.step_progress()
+
+                if apply_logical_op(db, json_data, self.flist) != self.invert:
+                    final_list.append(data)
 
         if user:
             user.end_progress()

From 2c0a2eaabced2cae2e0d0f2b3eba2179aaeac50a Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 20:56:12 -0500
Subject: [PATCH 15/46] Clean up

---
 gramps/gen/filters/_genericfilter.py | 51 +++++++++++++---------------
 1 file changed, 23 insertions(+), 28 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 045bc18226..056eb6707b 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -159,23 +159,30 @@ def apply_logical_op_to_all(
             user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
 
         def intersection(sets):
-            result = sets[0]
-            for s in sets[1:]:
-                result = result.intersection(s)
-            return result
+            if sets:
+                result = sets[0]
+                for s in sets[1:]:
+                    result = result.intersection(s)
+                return result
+            else:
+                return set()
+
+        # ------------------------------------------------------
+        # Optimiziation
+        # ------------------------------------------------------
+        all_maps = []
+        self.walk_filters(self, all_maps)
+        handles = None
+        # Get all positive non-inverted maps
+        for inverted, logical_op, maps in all_maps:
+            if logical_op == "and" and not inverted:
+                if handles is None:
+                    handles = intersection(maps)
+                else:
+                    handles = intersection([handles] + maps)
+        # ------------------------------------------------------
 
         if id_list is None:
-            # Optimiziation
-            all_maps = []
-            self.walk_filters(self, all_maps)
-            handles = None
-            # Get all positive non-inverted maps
-            for inverted, logical_op, maps in all_maps:
-                if logical_op == "and" and not inverted:
-                    if handles is None:
-                        handles = intersection(maps)
-                    else:
-                        handles = intersection([handles] + maps)
             if handles:
                 # Use these rather than going through entire database
                 for handle in handles:
@@ -199,25 +206,13 @@ def intersection(sets):
                             final_list.append(handle)
 
         else:
-            # Optimiziation
-            all_maps = []
-            self.walk_filters(self, all_maps)
-            handles = None
-            # Get all positive non-inverted maps
-            for inverted, logical_op, maps in all_maps:
-                if logical_op == "and" and not inverted:
-                    if handles is None:
-                        handles = intersection(maps)
-                    else:
-                        handles = intersection([handles] + maps)
-
             for data in id_list:
                 if tupleind is None:
                     handle = data
                 else:
                     handle = data[tupleind]
 
-                if handles is not None and handle not in handles:
+                if handles and handle not in handles:
                     continue
 
                 json_data = db.get_raw_person_data(handle)

From 923633ac5b2231a34c716b8ec47aabb37d3bea7c Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 07:44:01 -0500
Subject: [PATCH 16/46] Change all top-level apply to apply_to_all

---
 gramps/gen/proxy/filter.py                   |  6 +++---
 gramps/gui/editors/filtereditor.py           |  2 +-
 gramps/gui/views/treemodels/treebasemodel.py |  2 +-
 gramps/plugins/drawreport/calendarreport.py  |  2 +-
 gramps/plugins/drawreport/statisticschart.py |  2 +-
 gramps/plugins/drawreport/timeline.py        |  4 ++--
 gramps/plugins/gramplet/todogramplet.py      |  2 +-
 gramps/plugins/graph/gvrelgraph.py           |  2 +-
 gramps/plugins/lib/librecords.py             |  4 ++--
 gramps/plugins/quickview/samesurnames.py     |  4 ++--
 gramps/plugins/textreport/birthdayreport.py  |  2 +-
 gramps/plugins/textreport/familygroup.py     |  2 +-
 gramps/plugins/textreport/indivcomplete.py   |  2 +-
 gramps/plugins/textreport/placereport.py     |  2 +-
 gramps/plugins/textreport/tagreport.py       | 18 +++++++++---------
 gramps/plugins/tool/eventcmp.py              |  2 +-
 gramps/plugins/tool/removeunused.py          |  4 ++--
 gramps/plugins/tool/sortevents.py            |  2 +-
 gramps/plugins/view/geoevents.py             |  2 +-
 gramps/plugins/view/geoplaces.py             |  2 +-
 gramps/plugins/webreport/narrativeweb.py     |  2 +-
 gramps/plugins/webreport/webcal.py           |  2 +-
 22 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/gramps/gen/proxy/filter.py b/gramps/gen/proxy/filter.py
index d08f558113..4ea1820175 100644
--- a/gramps/gen/proxy/filter.py
+++ b/gramps/gen/proxy/filter.py
@@ -67,21 +67,21 @@ def __init__(
         self.person_filter = person_filter
         if person_filter:
             self.plist = set(
-                person_filter.apply(self.db, self.db.iter_person_handles(), user=user)
+                person_filter.apply_to_all(self.db, self.db.iter_person_handles(), user=user)
             )
         else:
             self.plist = set(self.db.iter_person_handles())
 
         if event_filter:
             self.elist = set(
-                event_filter.apply(self.db, self.db.iter_event_handles(), user=user)
+                event_filter.apply_to_all(self.db, self.db.iter_event_handles(), user=user)
             )
         else:
             self.elist = set(self.db.iter_event_handles())
 
         if note_filter:
             self.nlist = set(
-                note_filter.apply(self.db, self.db.iter_note_handles(), user=user)
+                note_filter.apply_to_all(self.db, self.db.iter_note_handles(), user=user)
             )
         else:
             self.nlist = set(self.db.iter_note_handles())
diff --git a/gramps/gui/editors/filtereditor.py b/gramps/gui/editors/filtereditor.py
index ae6eaf6970..293810d236 100644
--- a/gramps/gui/editors/filtereditor.py
+++ b/gramps/gui/editors/filtereditor.py
@@ -1312,7 +1312,7 @@ def test_clicked(self, obj):
         if node:
             filt = self.clist.get_object(node)
             try:
-                handle_list = filt.apply(self.db, self.get_all_handles())
+                handle_list = filt.apply_to_all(self.db, self.get_all_handles())
             except FilterError as msg:
                 (msg1, msg2) = msg.messages()
                 ErrorDialog(msg1, msg2, parent=self.window)
diff --git a/gramps/gui/views/treemodels/treebasemodel.py b/gramps/gui/views/treemodels/treebasemodel.py
index a217c46698..bfdafc956c 100644
--- a/gramps/gui/views/treemodels/treebasemodel.py
+++ b/gramps/gui/views/treemodels/treebasemodel.py
@@ -623,7 +623,7 @@ def __rebuild_filter(self, dfilter, skip, items, gen_cursor, data_map, add_func)
         assert not skip
         if dfilter:
             cdb = CacheProxyDb(self.db)
-            for handle in dfilter.apply(
+            for handle in dfilter.apply_to_all(
                 cdb,
                 tree=True,
                 user=User(parent=self.uistate.window, uistate=self.uistate),
diff --git a/gramps/plugins/drawreport/calendarreport.py b/gramps/plugins/drawreport/calendarreport.py
index f928e44bc2..3d99efe37f 100644
--- a/gramps/plugins/drawreport/calendarreport.py
+++ b/gramps/plugins/drawreport/calendarreport.py
@@ -351,7 +351,7 @@ def collect_data(self):
         and text.
         """
         people = self.database.iter_person_handles()
-        people = self.filter.apply(self.database, people, user=self._user)
+        people = self.filter.apply_to_all(self.database, people, user=self._user)
 
         with self._user.progress(
             _("Calendar Report"), _("Reading database..."), len(people)
diff --git a/gramps/plugins/drawreport/statisticschart.py b/gramps/plugins/drawreport/statisticschart.py
index 6e9ebac21f..fb0235d34a 100644
--- a/gramps/plugins/drawreport/statisticschart.py
+++ b/gramps/plugins/drawreport/statisticschart.py
@@ -907,7 +907,7 @@ def __init__(self, database, options, user):
             self._get_date(Date(year_to)),
         )
 
-        people = self.filter.apply(
+        people = self.filter.apply_to_all(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
diff --git a/gramps/plugins/drawreport/timeline.py b/gramps/plugins/drawreport/timeline.py
index 92d8c7f1c8..5b79591e8e 100644
--- a/gramps/plugins/drawreport/timeline.py
+++ b/gramps/plugins/drawreport/timeline.py
@@ -142,7 +142,7 @@ def __init__(self, database, options, user):
 
     def write_report(self):
         # Apply the filter
-        self.plist = self.filter.apply(
+        self.plist = self.filter.apply_to_all(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
@@ -385,7 +385,7 @@ def min_max_year(low, high, year):
 
     def name_size(self):
         """get the length of the name"""
-        self.plist = self.filter.apply(
+        self.plist = self.filter.apply_to_all(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
diff --git a/gramps/plugins/gramplet/todogramplet.py b/gramps/plugins/gramplet/todogramplet.py
index d51110e177..c768bea6b5 100644
--- a/gramps/plugins/gramplet/todogramplet.py
+++ b/gramps/plugins/gramplet/todogramplet.py
@@ -104,7 +104,7 @@ def get_note_list(self):
         FilterClass = GenericFilterFactory("Note")
         filter = FilterClass()
         filter.add_rule(rules.note.HasType(["To Do"]))
-        note_list = filter.apply(self.dbstate.db, all_notes)
+        note_list = filter.apply_to_all(self.dbstate.db, all_notes)
         return note_list
 
     def get_notes(self):
diff --git a/gramps/plugins/graph/gvrelgraph.py b/gramps/plugins/graph/gvrelgraph.py
index 567b2411ed..39ecc2919a 100644
--- a/gramps/plugins/graph/gvrelgraph.py
+++ b/gramps/plugins/graph/gvrelgraph.py
@@ -211,7 +211,7 @@ def __init__(self, database, options, user):
             self.advrelinfo = False
 
     def write_report(self):
-        person_handles = self._filter.apply(
+        person_handles = self._filter.apply_to_all(
             self._db, self._db.iter_person_handles(), user=self._user
         )
         # Hash people in a dictionary for faster inclusion checking
diff --git a/gramps/plugins/lib/librecords.py b/gramps/plugins/lib/librecords.py
index bbf42e1e98..8b706c3985 100644
--- a/gramps/plugins/lib/librecords.py
+++ b/gramps/plugins/lib/librecords.py
@@ -169,7 +169,7 @@ def get_unfiltered_person_from_handle(person_handle):
     person_handle_list = list(person_handle_list)
 
     if filter:
-        person_handle_list = filter.apply(db, person_handle_list, user=user)
+        person_handle_list = filter.apply_to_all(db, person_handle_list, user=user)
 
     for person_handle in person_handle_list:
         person = db.get_person_from_handle(person_handle)
@@ -386,7 +386,7 @@ def get_unfiltered_person_from_handle(person_handle):
         # Test if either father or mother are in filter
         if filter:
             # we don't want many progress reports popping up, so no user=user
-            if not filter.apply(db, [father_handle, mother_handle]):
+            if not filter.apply_to_all(db, [father_handle, mother_handle]):
                 continue
 
         father = db.get_person_from_handle(father_handle)
diff --git a/gramps/plugins/quickview/samesurnames.py b/gramps/plugins/quickview/samesurnames.py
index e8c3420b78..1b3c36c3fe 100644
--- a/gramps/plugins/quickview/samesurnames.py
+++ b/gramps/plugins/quickview/samesurnames.py
@@ -133,7 +133,7 @@ def run(database, document, person):
     else:
         rule = IncompleteSurname([])
     filter.add_rule(rule)
-    people = filter.apply(database, database.iter_person_handles())
+    people = filter.apply_to_all(database, database.iter_person_handles())
 
     matches = 0
     for person_handle in people:
@@ -183,7 +183,7 @@ def run_given(database, document, person):
     else:
         rule = IncompleteGiven([])
     filter.add_rule(rule)
-    people = filter.apply(database, database.iter_person_handles())
+    people = filter.apply_to_all(database, database.iter_person_handles())
 
     matches = 0
     for person_handle in people:
diff --git a/gramps/plugins/textreport/birthdayreport.py b/gramps/plugins/textreport/birthdayreport.py
index e0d10eef73..8eb4c4bdcc 100644
--- a/gramps/plugins/textreport/birthdayreport.py
+++ b/gramps/plugins/textreport/birthdayreport.py
@@ -291,7 +291,7 @@ def collect_data(self):
         and text.
         """
         people = self.database.iter_person_handles()
-        people = self.filter.apply(self.database, people, user=self._user)
+        people = self.filter.apply_to_all(self.database, people, user=self._user)
 
         ngettext = self._locale.translation.ngettext  # to see "nearby" comments
         rel_calc = get_relationship_calculator(reinit=True, clocale=self._locale)
diff --git a/gramps/plugins/textreport/familygroup.py b/gramps/plugins/textreport/familygroup.py
index a521a13af8..ed445ffcce 100644
--- a/gramps/plugins/textreport/familygroup.py
+++ b/gramps/plugins/textreport/familygroup.py
@@ -677,7 +677,7 @@ def write_report(self):
         if not self.filter:
             fam_list = flist
         else:
-            fam_list = self.filter.apply(self.db, flist, user=self._user)
+            fam_list = self.filter.apply_to_all(self.db, flist, user=self._user)
         if fam_list:
             with self._user.progress(
                 _("Family Group Report"), _("Writing families"), len(fam_list)
diff --git a/gramps/plugins/textreport/indivcomplete.py b/gramps/plugins/textreport/indivcomplete.py
index 57676585d6..d1120e6a24 100644
--- a/gramps/plugins/textreport/indivcomplete.py
+++ b/gramps/plugins/textreport/indivcomplete.py
@@ -850,7 +850,7 @@ def write_report(self):
         """write the report"""
         plist = self._db.get_person_handles(sort_handles=True, locale=self._locale)
         if self.filter:
-            ind_list = self.filter.apply(self._db, plist, user=self._user)
+            ind_list = self.filter.apply_to_all(self._db, plist, user=self._user)
         else:
             ind_list = plist
         if not ind_list:
diff --git a/gramps/plugins/textreport/placereport.py b/gramps/plugins/textreport/placereport.py
index 308e407d07..16d8c98918 100644
--- a/gramps/plugins/textreport/placereport.py
+++ b/gramps/plugins/textreport/placereport.py
@@ -125,7 +125,7 @@ def __init__(self, database, options, user):
         if self.filter.get_name() != "":
             # Use the selected filter to provide a list of place handles
             plist = self._db.iter_place_handles()
-            self.place_handles = self.filter.apply(self._db, plist, user=self._user)
+            self.place_handles = self.filter.apply_to_all(self._db, plist, user=self._user)
 
         if places:
             # Add places selected individually
diff --git a/gramps/plugins/textreport/tagreport.py b/gramps/plugins/textreport/tagreport.py
index a5af6ff737..6592f60a10 100644
--- a/gramps/plugins/textreport/tagreport.py
+++ b/gramps/plugins/textreport/tagreport.py
@@ -147,7 +147,7 @@ def write_people(self):
         filter_class = GenericFilterFactory("Person")
         a_filter = filter_class()
         a_filter.add_rule(rules.person.HasTag([self.tag]))
-        ind_list = a_filter.apply(self.database, plist)
+        ind_list = a_filter.apply_to_all(self.database, plist)
 
         if not ind_list:
             return
@@ -235,7 +235,7 @@ def write_families(self):
         filter_class = GenericFilterFactory("Family")
         a_filter = filter_class()
         a_filter.add_rule(rules.family.HasTag([self.tag]))
-        fam_list = a_filter.apply(self.database, flist)
+        fam_list = a_filter.apply_to_all(self.database, flist)
 
         if not fam_list:
             return
@@ -324,7 +324,7 @@ def write_events(self):
         filter_class = GenericFilterFactory("Event")
         a_filter = filter_class()
         a_filter.add_rule(rules.event.HasTag([self.tag]))
-        event_list = a_filter.apply(self.database, elist)
+        event_list = a_filter.apply_to_all(self.database, elist)
 
         if not event_list:
             return
@@ -406,7 +406,7 @@ def write_places(self):
         filter_class = GenericFilterFactory("Place")
         a_filter = filter_class()
         a_filter.add_rule(rules.place.HasTag([self.tag]))
-        place_list = a_filter.apply(self.database, plist)
+        place_list = a_filter.apply_to_all(self.database, plist)
 
         if not place_list:
             return
@@ -487,7 +487,7 @@ def write_notes(self):
         filter_class = GenericFilterFactory("Note")
         a_filter = filter_class()
         a_filter.add_rule(rules.note.HasTag([self.tag]))
-        note_list = a_filter.apply(self.database, nlist)
+        note_list = a_filter.apply_to_all(self.database, nlist)
 
         if not note_list:
             return
@@ -559,7 +559,7 @@ def write_media(self):
         filter_class = GenericFilterFactory("Media")
         a_filter = filter_class()
         a_filter.add_rule(rules.media.HasTag([self.tag]))
-        media_list = a_filter.apply(self.database, mlist)
+        media_list = a_filter.apply_to_all(self.database, mlist)
 
         if not media_list:
             return
@@ -643,7 +643,7 @@ def write_repositories(self):
         filter_class = GenericFilterFactory("Repository")
         a_filter = filter_class()
         a_filter.add_rule(rules.repository.HasTag([self.tag]))
-        repo_list = a_filter.apply(self.database, rlist)
+        repo_list = a_filter.apply_to_all(self.database, rlist)
 
         if not repo_list:
             return
@@ -728,7 +728,7 @@ def write_sources(self):
         filter_class = GenericFilterFactory("Source")
         a_filter = filter_class()
         a_filter.add_rule(rules.source.HasTag([self.tag]))
-        source_list = a_filter.apply(self.database, slist)
+        source_list = a_filter.apply_to_all(self.database, slist)
 
         if not source_list:
             return
@@ -810,7 +810,7 @@ def write_citations(self):
         filter_class = GenericFilterFactory("Citation")
         a_filter = filter_class()
         a_filter.add_rule(rules.citation.HasTag([self.tag]))
-        citation_list = a_filter.apply(self.database, clist)
+        citation_list = a_filter.apply_to_all(self.database, clist)
 
         if not citation_list:
             return
diff --git a/gramps/plugins/tool/eventcmp.py b/gramps/plugins/tool/eventcmp.py
index fff023f2a6..69482abc86 100644
--- a/gramps/plugins/tool/eventcmp.py
+++ b/gramps/plugins/tool/eventcmp.py
@@ -184,7 +184,7 @@ def on_apply_clicked(self, obj):
         progress_bar = ProgressMeter(_("Comparing events"), "", parent=self.window)
         progress_bar.set_pass(_("Selecting people"), 1)
 
-        plist = cfilter.apply(self.db, self.db.iter_person_handles())
+        plist = cfilter.apply_to_all(self.db, self.db.iter_person_handles())
 
         progress_bar.step()
         progress_bar.close()
diff --git a/gramps/plugins/tool/removeunused.py b/gramps/plugins/tool/removeunused.py
index c687f60263..a3fdecc8fc 100644
--- a/gramps/plugins/tool/removeunused.py
+++ b/gramps/plugins/tool/removeunused.py
@@ -305,10 +305,10 @@ def collect_unused(self):
         FilterClass = GenericFilterFactory("Note")
         filter1 = FilterClass()
         filter1.add_rule(rules.note.HasType(["To Do"]))
-        todo_list = filter1.apply(self.dbstate.db, all_notes)
+        todo_list = filter1.apply_to_all(self.dbstate.db, all_notes)
         filter2 = FilterClass()
         filter2.add_rule(rules.note.HasType(["Link"]))
-        link_list = filter2.apply(self.dbstate.db, all_notes)
+        link_list = filter2.apply_to_all(self.dbstate.db, all_notes)
 
         for the_type, cursor_func, total_func in tables:
             if not self.options.handler.options_dict[the_type]:
diff --git a/gramps/plugins/tool/sortevents.py b/gramps/plugins/tool/sortevents.py
index 3a939f001a..55bee61052 100644
--- a/gramps/plugins/tool/sortevents.py
+++ b/gramps/plugins/tool/sortevents.py
@@ -115,7 +115,7 @@ def sort_person_events(self, trans):
         """
         Sort the personal events associated with the selected people.
         """
-        people_handles = self.filter.apply(
+        people_handles = self.filter.apply_to_all(
             self.db, self.db.iter_person_handles(), user=self._user
         )
         self.progress.set_pass(
diff --git a/gramps/plugins/view/geoevents.py b/gramps/plugins/view/geoevents.py
index 565458d87e..939574cedb 100644
--- a/gramps/plugins/view/geoevents.py
+++ b/gramps/plugins/view/geoevents.py
@@ -371,7 +371,7 @@ def _createmap(self, obj):
             progress.close()
         elif self.generic_filter:
             user = self.uistate.viewmanager.user
-            events_list = self.generic_filter.apply(dbstate.db, user=user)
+            events_list = self.generic_filter.apply_to_all(dbstate.db, user=user)
             progress = ProgressMeter(
                 self.window_name, can_cancel=False, parent=self.uistate.window
             )
diff --git a/gramps/plugins/view/geoplaces.py b/gramps/plugins/view/geoplaces.py
index 6fd9e94ea3..a74dc503ff 100644
--- a/gramps/plugins/view/geoplaces.py
+++ b/gramps/plugins/view/geoplaces.py
@@ -422,7 +422,7 @@ def _createmap(self, place_x):
             progress.close()
         elif self.generic_filter:
             user = self.uistate.viewmanager.user
-            place_list = self.generic_filter.apply(dbstate.db, user=user)
+            place_list = self.generic_filter.apply_to_all(dbstate.db, user=user)
             progress = ProgressMeter(
                 self.window_name, can_cancel=False, parent=self.uistate.window
             )
diff --git a/gramps/plugins/webreport/narrativeweb.py b/gramps/plugins/webreport/narrativeweb.py
index 0f33138b49..ae427ad8e5 100644
--- a/gramps/plugins/webreport/narrativeweb.py
+++ b/gramps/plugins/webreport/narrativeweb.py
@@ -643,7 +643,7 @@ def _build_obj_dict(self):
             self.obj_dict[obj_class] = defaultdict(set)
 
         ind_list = self._db.iter_person_handles()
-        ind_list = self.filter.apply(self._db, ind_list, user=self.user)
+        ind_list = self.filter.apply_to_all(self._db, ind_list, user=self.user)
 
         message = _("Constructing list of other objects...")
         pgr_title = self.pgrs_title(None)
diff --git a/gramps/plugins/webreport/webcal.py b/gramps/plugins/webreport/webcal.py
index d9170e5cd0..62a2d0fa6e 100644
--- a/gramps/plugins/webreport/webcal.py
+++ b/gramps/plugins/webreport/webcal.py
@@ -1412,7 +1412,7 @@ def collect_data(self, this_year):
         db = self.database
 
         people = db.iter_person_handles()
-        people = self.filter.apply(db, people, user=self._user)
+        people = self.filter.apply_to_all(db, people, user=self._user)
 
         with self._user.progress(
             _("Web Calendar Report"), _("Reading database..."), len(people)

From 8d1a4f788d8b7a3866c37c6cfecfbeea27f0c97c Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 08:47:41 -0500
Subject: [PATCH 17/46] Linting

---
 gramps/gen/filters/rules/_hastagbase.py      |  8 ++++++--
 gramps/gen/proxy/filter.py                   | 12 +++++++++---
 gramps/gui/views/treemodels/flatbasemodel.py |  4 +++-
 gramps/plugins/db/dbapi/dbapi.py             |  2 +-
 gramps/plugins/textreport/placereport.py     |  4 +++-
 5 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/gramps/gen/filters/rules/_hastagbase.py b/gramps/gen/filters/rules/_hastagbase.py
index 73a1c5effe..0451dba66a 100644
--- a/gramps/gen/filters/rules/_hastagbase.py
+++ b/gramps/gen/filters/rules/_hastagbase.py
@@ -61,14 +61,18 @@ def prepare(self, db, user):
         tag = db.get_tag_from_name(self.list[0])
         if tag is not None:
             self.tag_handle = tag.get_handle()
-            results = db.select(self.table, ["$.handle"], ("$.tag_list", "LIKE", f'%"{self.tag_handle}"%'))
+            results = db.select(
+                self.table,
+                ["$.handle"],
+                ("$.tag_list", "LIKE", f'%"{self.tag_handle}"%'),
+            )
             self.map = set([row["handle"] for row in list(results)])
         else:
             self.map = set()
 
     def get_rules_with_maps(self):
         return [self]
-            
+
     def apply_to_one(self, db, data):
         """
         Apply the rule.  Return True for a match.
diff --git a/gramps/gen/proxy/filter.py b/gramps/gen/proxy/filter.py
index 4ea1820175..5414e3fcfc 100644
--- a/gramps/gen/proxy/filter.py
+++ b/gramps/gen/proxy/filter.py
@@ -67,21 +67,27 @@ def __init__(
         self.person_filter = person_filter
         if person_filter:
             self.plist = set(
-                person_filter.apply_to_all(self.db, self.db.iter_person_handles(), user=user)
+                person_filter.apply_to_all(
+                    self.db, self.db.iter_person_handles(), user=user
+                )
             )
         else:
             self.plist = set(self.db.iter_person_handles())
 
         if event_filter:
             self.elist = set(
-                event_filter.apply_to_all(self.db, self.db.iter_event_handles(), user=user)
+                event_filter.apply_to_all(
+                    self.db, self.db.iter_event_handles(), user=user
+                )
             )
         else:
             self.elist = set(self.db.iter_event_handles())
 
         if note_filter:
             self.nlist = set(
-                note_filter.apply_to_all(self.db, self.db.iter_note_handles(), user=user)
+                note_filter.apply_to_all(
+                    self.db, self.db.iter_note_handles(), user=user
+                )
             )
         else:
             self.nlist = set(self.db.iter_note_handles())
diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py
index 1c7ae9b33a..5f7c1e1b93 100644
--- a/gramps/gui/views/treemodels/flatbasemodel.py
+++ b/gramps/gui/views/treemodels/flatbasemodel.py
@@ -627,7 +627,9 @@ def _rebuild_filter(self, ignore=None):
             if self.search:
                 ident = False
                 if ignore is None:
-                    dlist = self.search.apply_to_all(cdb, allkeys, tupleind=1, user=self.user)
+                    dlist = self.search.apply_to_all(
+                        cdb, allkeys, tupleind=1, user=self.user
+                    )
                 else:
                     dlist = self.search.apply_to_all(
                         cdb, [k for k in allkeys if k[1] != ignore], tupleind=1
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 3a0e1acb97..ab747c93dc 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -1295,7 +1295,7 @@ def select(
 
         self.dbapi.execute(
             f"select {select_clause} from {table}{where_clause}{sort_by_clause}{offset_limit_clause};",
-            values
+            values,
         )
         for row in self.dbapi.fetchall():
             if "$" in selections:
diff --git a/gramps/plugins/textreport/placereport.py b/gramps/plugins/textreport/placereport.py
index 16d8c98918..147b42990c 100644
--- a/gramps/plugins/textreport/placereport.py
+++ b/gramps/plugins/textreport/placereport.py
@@ -125,7 +125,9 @@ def __init__(self, database, options, user):
         if self.filter.get_name() != "":
             # Use the selected filter to provide a list of place handles
             plist = self._db.iter_place_handles()
-            self.place_handles = self.filter.apply_to_all(self._db, plist, user=self._user)
+            self.place_handles = self.filter.apply_to_all(
+                self._db, plist, user=self._user
+            )
 
         if places:
             # Add places selected individually

From c031cfb50c7b930ce68918fbb2a49ecdeb0f5361 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 14:33:46 -0500
Subject: [PATCH 18/46] Add protections on loggin

---
 gramps/gen/lib/serialize.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/gramps/gen/lib/serialize.py b/gramps/gen/lib/serialize.py
index 8a5a11db69..526026e3aa 100644
--- a/gramps/gen/lib/serialize.py
+++ b/gramps/gen/lib/serialize.py
@@ -115,7 +115,7 @@ class BlobSerializer:
 
     @staticmethod
     def data_to_object(obj_class, data):
-        LOG.debug("blob, data_to_object: %s(%r)", obj_class, data[0])
+        LOG.debug("blob, data_to_object: %s(%r)", obj_class, data[0] if data else data)
         return obj_class.create(data)
 
     @staticmethod
@@ -135,7 +135,7 @@ def object_to_string(obj):
 
     @staticmethod
     def data_to_string(data):
-        LOG.debug("blob, data_to_string: %s...", data[:2])
+        LOG.debug("blob, data_to_string: %s...", data[0] if data else data)
         return pickle.dumps(data)
 
     @staticmethod
@@ -160,7 +160,10 @@ class JSONSerializer:
 
     @staticmethod
     def data_to_object(obj_class, data):
-        LOG.debug("json, data_to_object: {'_class': %r, ...}", data["_class"])
+        LOG.debug(
+            "json, data_to_object: {'_class': %r, ...}",
+            data["_class"] if (data and "_class" in data) else data,
+        )
         return from_dict(data)
 
     @staticmethod
@@ -180,7 +183,10 @@ def object_to_string(obj):
 
     @staticmethod
     def data_to_string(data):
-        LOG.debug("json, data_to_string: {'_class': %r, ...}", data["_class"])
+        LOG.debug(
+            "json, data_to_string: {'_class': %r, ...}",
+            data["_class"] if (data and "_class" in data) else data,
+        )
         return json.dumps(data)
 
     @staticmethod

From dfbbb20686dda33614be4108a2449b93bb87041a Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 08:54:30 -0500
Subject: [PATCH 19/46] Linting

---
 gramps/gen/filters/_genericfilter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 056eb6707b..59b3e897d9 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -195,8 +195,8 @@ def intersection(sets):
                         final_list.append(json_data)
 
             else:
-                with self.get_tree_cursor(db) if tree else self.get_cursor(
-                    db
+                with (
+                        self.get_tree_cursor(db) if tree else self.get_cursor(db)
                 ) as cursor:
                     for handle, data in cursor:
                         if user:

From f04ac4e5f5377e1c36c0a507aa42afe25ac005bd Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 08:56:00 -0500
Subject: [PATCH 20/46] Linting

---
 gramps/gen/filters/_genericfilter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 59b3e897d9..7c72575b5a 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -196,7 +196,7 @@ def intersection(sets):
 
             else:
                 with (
-                        self.get_tree_cursor(db) if tree else self.get_cursor(db)
+                    self.get_tree_cursor(db) if tree else self.get_cursor(db)
                 ) as cursor:
                     for handle, data in cursor:
                         if user:

From 40322c994d9107b92d878adfa46d56eda634477d Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 11:45:36 -0500
Subject: [PATCH 21/46] First rough pass at all filters

---
 gramps/gen/filters/_genericfilter.py          |  6 +++--
 gramps/gen/filters/_paramfilter.py            |  4 +--
 gramps/gen/filters/rules/_changedsincebase.py |  4 +--
 gramps/gen/filters/rules/_everything.py       |  2 +-
 gramps/gen/filters/rules/_hasattributebase.py |  3 ++-
 gramps/gen/filters/rules/_hascitationbase.py  |  3 ++-
 gramps/gen/filters/rules/_haseventbase.py     |  3 ++-
 gramps/gen/filters/rules/_hasgallerybase.py   |  4 +--
 gramps/gen/filters/rules/_hasgrampsid.py      |  4 +--
 gramps/gen/filters/rules/_hasldsbase.py       |  3 ++-
 gramps/gen/filters/rules/_hasnotebase.py      |  4 +--
 gramps/gen/filters/rules/_hasnoteregexbase.py |  4 +--
 .../gen/filters/rules/_hasnotesubstrbase.py   |  4 +--
 .../filters/rules/_hasreferencecountbase.py   |  8 +++---
 gramps/gen/filters/rules/_hassourcebase.py    |  3 ++-
 .../gen/filters/rules/_hassourcecountbase.py  |  3 ++-
 gramps/gen/filters/rules/_hassourceofbase.py  |  3 ++-
 gramps/gen/filters/rules/_hastagbase.py       |  3 ---
 gramps/gen/filters/rules/_isprivate.py        |  4 +--
 gramps/gen/filters/rules/_ispublic.py         |  4 +--
 .../filters/rules/_matcheseventfilterbase.py  |  5 ++--
 .../gen/filters/rules/_matchesfilterbase.py   |  1 -
 .../rules/_matchessourceconfidencebase.py     |  3 ++-
 .../filters/rules/_matchessourcefilterbase.py |  6 +++--
 gramps/gen/filters/rules/_regexpidbase.py     |  4 +--
 gramps/gen/filters/rules/_rule.py             |  9 +++++++
 .../filters/rules/citation/_hascitation.py    |  3 ++-
 .../gen/filters/rules/citation/_hassource.py  |  5 ++--
 .../filters/rules/citation/_hassourceidof.py  |  6 +++--
 .../rules/citation/_hassourcenoteregexp.py    |  6 +++--
 gramps/gen/filters/rules/citation/_hastag.py  |  1 +
 .../rules/citation/_matchespagesubstringof.py |  4 +--
 .../citation/_matchesrepositoryfilter.py      |  3 ++-
 .../rules/citation/_matchessourcefilter.py    |  6 ++---
 .../rules/citation/_regexpsourceidof.py       |  6 +++--
 gramps/gen/filters/rules/event/_hasdata.py    |  3 ++-
 .../gen/filters/rules/event/_hasdayofweek.py  |  3 ++-
 gramps/gen/filters/rules/event/_hastag.py     |  1 +
 gramps/gen/filters/rules/event/_hastype.py    |  6 +++--
 .../rules/event/_matchespersonfilter.py       | 27 ++++++++++++-------
 .../rules/event/_matchesplacefilter.py        |  6 +++--
 .../gen/filters/rules/family/_childhasidof.py |  2 +-
 .../filters/rules/family/_childhasnameof.py   |  2 +-
 .../filters/rules/family/_fatherhasidof.py    |  2 +-
 .../filters/rules/family/_fatherhasnameof.py  |  2 +-
 gramps/gen/filters/rules/family/_hasevent.py  |  3 ++-
 gramps/gen/filters/rules/family/_hasnote.py   |  1 +
 .../gen/filters/rules/family/_hasreltype.py   |  3 ++-
 gramps/gen/filters/rules/family/_hastag.py    |  1 +
 gramps/gen/filters/rules/family/_hastwins.py  |  3 ++-
 .../gen/filters/rules/family/_isancestorof.py |  4 +--
 .../gen/filters/rules/family/_isbookmarked.py |  4 +--
 .../filters/rules/family/_isdescendantof.py   |  4 +--
 .../gen/filters/rules/family/_memberbase.py   |  9 ++++---
 .../filters/rules/family/_motherhasidof.py    |  2 +-
 .../filters/rules/family/_motherhasnameof.py  |  2 +-
 .../filters/rules/family/_regexpchildname.py  |  2 +-
 .../filters/rules/family/_regexpfathername.py |  2 +-
 .../filters/rules/family/_regexpmothername.py |  2 +-
 .../filters/rules/family/_searchchildname.py  |  2 +-
 .../filters/rules/family/_searchfathername.py |  2 +-
 .../filters/rules/family/_searchmothername.py |  2 +-
 gramps/gen/filters/rules/media/_hasmedia.py   |  3 ++-
 gramps/gen/filters/rules/note/_hasnote.py     |  3 ++-
 gramps/gen/filters/rules/note/_hastype.py     |  3 ++-
 .../filters/rules/note/_matchesregexpof.py    |  3 ++-
 .../filters/rules/note/_matchessubstringof.py |  3 ++-
 .../person/_deeprelationshippathbetween.py    |  8 +++---
 .../gen/filters/rules/person/_disconnected.py |  6 ++---
 gramps/gen/filters/rules/person/_everyone.py  |  2 +-
 .../person/_familywithincompleteevent.py      |  3 ++-
 .../gen/filters/rules/person/_hasaddress.py   |  4 +--
 .../filters/rules/person/_hasaddresstext.py   |  3 ++-
 .../filters/rules/person/_hasalternatename.py |  4 +--
 .../filters/rules/person/_hasassociation.py   |  4 +--
 gramps/gen/filters/rules/person/_hasbirth.py  |  3 ++-
 .../rules/person/_hascommonancestorwith.py    |  5 ++--
 .../_hascommonancestorwithfiltermatch.py      | 12 +++++----
 gramps/gen/filters/rules/person/_hasdeath.py  |  3 ++-
 gramps/gen/filters/rules/person/_hasevent.py  |  3 ++-
 .../rules/person/_hasfamilyattribute.py       |  3 ++-
 .../filters/rules/person/_hasfamilyevent.py   |  3 ++-
 gramps/gen/filters/rules/person/_hasnameof.py |  3 ++-
 .../rules/person/_hasnameorigintype.py        |  3 ++-
 .../gen/filters/rules/person/_hasnametype.py  |  3 ++-
 .../gen/filters/rules/person/_hasnickname.py  |  3 ++-
 .../filters/rules/person/_hasothergender.py   |  4 +--
 .../filters/rules/person/_hasrelationship.py  |  3 ++-
 .../filters/rules/person/_hassoundexname.py   |  3 ++-
 .../person/_hastextmatchingsubstringof.py     |  3 ++-
 .../filters/rules/person/_hasunknowngender.py |  4 +--
 .../filters/rules/person/_havealtfamilies.py  |  3 ++-
 .../gen/filters/rules/person/_havechildren.py |  3 ++-
 .../filters/rules/person/_incompletenames.py  |  3 ++-
 .../rules/person/_isancestoroffiltermatch.py  | 10 ++++---
 .../gen/filters/rules/person/_isbookmarked.py |  4 +--
 .../rules/person/_ischildoffiltermatch.py     |  4 +--
 .../filters/rules/person/_isdefaultperson.py  |  7 +++--
 .../rules/person/_isdescendantfamilyof.py     | 18 ++++++-------
 .../filters/rules/person/_isdescendantof.py   |  4 +--
 .../person/_isdescendantoffiltermatch.py      |  4 +--
 .../rules/person/_isduplicatedancestorof.py   | 14 +++++-----
 gramps/gen/filters/rules/person/_isfemale.py  |  4 +--
 .../_islessthannthgenerationancestorof.py     |  4 +--
 ...ssthannthgenerationancestorofbookmarked.py |  9 +++----
 ...hannthgenerationancestorofdefaultperson.py |  7 +++--
 .../_islessthannthgenerationdescendantof.py   |  4 +--
 gramps/gen/filters/rules/person/_ismale.py    |  4 +--
 .../_ismorethannthgenerationancestorof.py     |  4 +--
 .../_ismorethannthgenerationdescendantof.py   |  4 +--
 .../rules/person/_isparentoffiltermatch.py    |  4 +--
 .../filters/rules/person/_isrelatedwith.py    | 12 ++++-----
 .../rules/person/_issiblingoffiltermatch.py   |  4 +--
 .../rules/person/_isspouseoffiltermatch.py    |  3 ++-
 gramps/gen/filters/rules/person/_iswitness.py |  3 ++-
 gramps/gen/filters/rules/person/_matchidof.py |  4 +--
 .../filters/rules/person/_missingparent.py    |  3 ++-
 .../rules/person/_multiplemarriages.py        |  4 +--
 .../gen/filters/rules/person/_nevermarried.py |  4 +--
 .../gen/filters/rules/person/_nobirthdate.py  |  4 ++-
 .../gen/filters/rules/person/_nodeathdate.py  |  3 ++-
 .../person/_personwithincompleteevent.py      |  3 ++-
 .../filters/rules/person/_probablyalive.py    |  3 ++-
 .../gen/filters/rules/person/_regexpname.py   |  3 ++-
 .../rules/person/_relationshippathbetween.py  |  4 +--
 .../_relationshippathbetweenbookmarks.py      |  4 +--
 .../gen/filters/rules/person/_searchname.py   |  3 ++-
 gramps/gen/filters/rules/place/_hasdata.py    |  3 ++-
 .../gen/filters/rules/place/_hasnolatorlon.py |  4 +--
 gramps/gen/filters/rules/place/_hasplace.py   |  3 ++-
 gramps/gen/filters/rules/place/_hastitle.py   |  3 ++-
 .../rules/place/_inlatlonneighborhood.py      |  4 ++-
 .../gen/filters/rules/place/_isenclosedby.py  |  6 ++---
 .../rules/place/_matcheseventfilter.py        |  7 ++---
 gramps/gen/filters/rules/place/_withinarea.py |  8 +++---
 .../gen/filters/rules/repository/_hasrepo.py  |  4 ++-
 .../repository/_matchesnamesubstringof.py     |  4 +--
 .../filters/rules/source/_hasrepository.py    |  4 +--
 .../source/_hasrepositorycallnumberref.py     |  6 ++---
 .../rules/source/_matchesrepositoryfilter.py  |  7 ++---
 .../rules/source/_matchestitlesubstringof.py  |  4 +--
 141 files changed, 347 insertions(+), 258 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 7c72575b5a..bf5164d80e 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -151,7 +151,7 @@ def walk_filters(self, filter, result):
         if rules:
             result.append((filter.invert, filter.logical_op, rules))
 
-    def apply_logical_op_to_all(
+    def apply_to_one_logical_op_to_all(
         self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
     ):
         final_list = []
@@ -260,7 +260,9 @@ def apply_to_one(self, db, data):
             raise Exception("invalid operator: %r" % self.logical_op)
         return res != self.invert
 
-    def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
+    def apply_to_one_to_all(
+        self, db, id_list=None, tupleind=None, user=None, tree=False
+    ):
         """
         Apply the filter using db.
         If id_list given, the handles in id_list are used. If not given
diff --git a/gramps/gen/filters/_paramfilter.py b/gramps/gen/filters/_paramfilter.py
index ebcd5d2a71..50a141390d 100644
--- a/gramps/gen/filters/_paramfilter.py
+++ b/gramps/gen/filters/_paramfilter.py
@@ -45,7 +45,7 @@ def __init__(self, source=None):
     def set_parameter(self, param):
         self.param_list = [param]
 
-    def apply(self, db, id_list=None, user=None):
+    def apply_to_all(self, db, id_list=None, user=None):
         for rule in self.flist:
             # rule.set_list(self.param_list)
             #
@@ -62,7 +62,7 @@ def apply(self, db, id_list=None, user=None):
                     "Custom filters can not twice be used" " in a parameter filter"
                 )
             rule.requestprepare(db, user)
-        result = GenericFilter.apply(self, db, id_list)
+        result = GenericFilter.apply_to_all(self, db, id_list)
         for rule in self.flist:
             rule.requestreset()
         return result
diff --git a/gramps/gen/filters/rules/_changedsincebase.py b/gramps/gen/filters/rules/_changedsincebase.py
index 3cc57b1ed9..5e1a955f78 100644
--- a/gramps/gen/filters/rules/_changedsincebase.py
+++ b/gramps/gen/filters/rules/_changedsincebase.py
@@ -101,8 +101,8 @@ def prepare(self, db, user):
         if self.list[1]:
             self.before = self.time_str_to_sec(self.list[1])
 
-    def apply(self, db, obj):
-        obj_time = obj.get_change_time()
+    def apply_to_one(self, db, data):
+        obj_time = data["change"]
         if self.since:
             if obj_time < self.since:
                 return False
diff --git a/gramps/gen/filters/rules/_everything.py b/gramps/gen/filters/rules/_everything.py
index 3ec6648436..6bca78ad56 100644
--- a/gramps/gen/filters/rules/_everything.py
+++ b/gramps/gen/filters/rules/_everything.py
@@ -50,5 +50,5 @@ class Everything(Rule):
     def is_empty(self):
         return True
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         return True
diff --git a/gramps/gen/filters/rules/_hasattributebase.py b/gramps/gen/filters/rules/_hasattributebase.py
index 1b29aa2530..4063928a26 100644
--- a/gramps/gen/filters/rules/_hasattributebase.py
+++ b/gramps/gen/filters/rules/_hasattributebase.py
@@ -63,10 +63,11 @@ def prepare(self, db, user):
             self.attribute_type = AttributeType()
             self.attribute_type.set_from_xml_str(self.list[0])
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True if a match.
         """
+        obj = self.get_object(data)
         if self.attribute_type:
             for attribute in obj.get_attribute_list():
                 name_match = attribute.get_type() == self.attribute_type
diff --git a/gramps/gen/filters/rules/_hascitationbase.py b/gramps/gen/filters/rules/_hascitationbase.py
index 0d2c0cfcff..352e3682f5 100644
--- a/gramps/gen/filters/rules/_hascitationbase.py
+++ b/gramps/gen/filters/rules/_hascitationbase.py
@@ -62,7 +62,8 @@ def prepare(self, db, user):
         except:
             pass
 
-    def apply(self, dbase, object):
+    def apply_to_one(self, dbase, data):
+        object = self.get_object(data)
         for citation_handle in object.get_citation_list():
             citation = dbase.get_citation_from_handle(citation_handle)
             if self._apply(dbase, citation):
diff --git a/gramps/gen/filters/rules/_haseventbase.py b/gramps/gen/filters/rules/_haseventbase.py
index dff30f83cb..40c63bbe59 100644
--- a/gramps/gen/filters/rules/_haseventbase.py
+++ b/gramps/gen/filters/rules/_haseventbase.py
@@ -71,10 +71,11 @@ def prepare(self, db, user):
         except:
             pass
 
-    def apply(self, db, event):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True if a match.
         """
+        event = self.get_object(data)
         if self.event_type:
             if self.event_type.is_custom() and self.use_regex:
                 if self.regex[0].search(str(event.type)) is None:
diff --git a/gramps/gen/filters/rules/_hasgallerybase.py b/gramps/gen/filters/rules/_hasgallerybase.py
index 5bf5c61bee..b2565bc9a1 100644
--- a/gramps/gen/filters/rules/_hasgallerybase.py
+++ b/gramps/gen/filters/rules/_hasgallerybase.py
@@ -60,8 +60,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, obj):
-        count = len(obj.get_media_list())
+    def apply_to_one(self, db, data):
+        count = len(data["media_list"])
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
         elif self.count_type == 2:  # "greater than"
diff --git a/gramps/gen/filters/rules/_hasgrampsid.py b/gramps/gen/filters/rules/_hasgrampsid.py
index fbfb5a8262..4c8c1b1089 100644
--- a/gramps/gen/filters/rules/_hasgrampsid.py
+++ b/gramps/gen/filters/rules/_hasgrampsid.py
@@ -49,9 +49,9 @@ class HasGrampsId(Rule):
     description = "Matches objects with a specified Gramps ID"
     category = _("General filters")
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         """
         apply the rule on the obj.
         return true if the rule passes, false otherwise.
         """
-        return obj.gramps_id == self.list[0]
+        return data["gramps_id"] == self.list[0]
diff --git a/gramps/gen/filters/rules/_hasldsbase.py b/gramps/gen/filters/rules/_hasldsbase.py
index 8e29ed2c4f..df9cb79ede 100644
--- a/gramps/gen/filters/rules/_hasldsbase.py
+++ b/gramps/gen/filters/rules/_hasldsbase.py
@@ -63,7 +63,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
+        obj = self.get_obj(data)
         count = len(obj.get_lds_ord_list())
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
diff --git a/gramps/gen/filters/rules/_hasnotebase.py b/gramps/gen/filters/rules/_hasnotebase.py
index 441bc582e0..bbd9a00333 100644
--- a/gramps/gen/filters/rules/_hasnotebase.py
+++ b/gramps/gen/filters/rules/_hasnotebase.py
@@ -69,8 +69,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, obj):
-        count = len(obj.get_note_list())
+    def apply_to_one(self, db, data):
+        count = len(data["note_list"])
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
         elif self.count_type == 2:  # "greater than"
diff --git a/gramps/gen/filters/rules/_hasnoteregexbase.py b/gramps/gen/filters/rules/_hasnoteregexbase.py
index 50677efd96..38018bc7c0 100644
--- a/gramps/gen/filters/rules/_hasnoteregexbase.py
+++ b/gramps/gen/filters/rules/_hasnoteregexbase.py
@@ -51,8 +51,8 @@ class HasNoteRegexBase(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, person):
-        for handle in person.get_note_list():
+    def apply_to_one(self, db, data):
+        for handle in data["note_list"]:
             note = db.get_note_from_handle(handle)
             if self.match_substring(0, note.get()):
                 return True
diff --git a/gramps/gen/filters/rules/_hasnotesubstrbase.py b/gramps/gen/filters/rules/_hasnotesubstrbase.py
index 25b3c3fc87..f308e5dd06 100644
--- a/gramps/gen/filters/rules/_hasnotesubstrbase.py
+++ b/gramps/gen/filters/rules/_hasnotesubstrbase.py
@@ -46,8 +46,8 @@ class HasNoteSubstrBase(Rule):
     description = "Matches objects whose notes contain text matching a " "substring"
     category = _("General filters")
 
-    def apply(self, db, person):
-        notelist = person.get_note_list()
+    def apply_to_one(self, db, data):
+        notelist = data["note_list"]
         for notehandle in notelist:
             note = db.get_note_from_handle(notehandle)
             n = note.get()
diff --git a/gramps/gen/filters/rules/_hasreferencecountbase.py b/gramps/gen/filters/rules/_hasreferencecountbase.py
index c8a6e2977d..f8ec74f25e 100644
--- a/gramps/gen/filters/rules/_hasreferencecountbase.py
+++ b/gramps/gen/filters/rules/_hasreferencecountbase.py
@@ -57,11 +57,9 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[1])
 
-    def apply(self, db, obj):
-        handle = obj.get_handle()
-        count = 0
-        for item in db.find_backlink_handles(handle):
-            count += 1
+    def apply_to_one(self, db, data):
+        handle = data["handle"]
+        count = len(list(db.find_backlink_handles(handle)))
 
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
diff --git a/gramps/gen/filters/rules/_hassourcebase.py b/gramps/gen/filters/rules/_hassourcebase.py
index 024143571a..8394ad5f20 100644
--- a/gramps/gen/filters/rules/_hassourcebase.py
+++ b/gramps/gen/filters/rules/_hassourcebase.py
@@ -50,7 +50,8 @@ class HasSourceBase(Rule):
     category = _("Citation/source filters")
     allow_regex = True
 
-    def apply(self, db, source):
+    def apply_to_one(self, db, data):
+        source = self.get_object(data)
         if not self.match_substring(0, source.get_title()):
             return False
 
diff --git a/gramps/gen/filters/rules/_hassourcecountbase.py b/gramps/gen/filters/rules/_hassourcecountbase.py
index 8b4aab709b..459bc26bb1 100644
--- a/gramps/gen/filters/rules/_hassourcecountbase.py
+++ b/gramps/gen/filters/rules/_hassourcecountbase.py
@@ -63,7 +63,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
+        obj = self.get_object(data)
         count = len(obj.get_citation_list())
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
diff --git a/gramps/gen/filters/rules/_hassourceofbase.py b/gramps/gen/filters/rules/_hassourceofbase.py
index 4e04be727f..44f05533cb 100644
--- a/gramps/gen/filters/rules/_hassourceofbase.py
+++ b/gramps/gen/filters/rules/_hassourceofbase.py
@@ -61,7 +61,8 @@ def prepare(self, db, user):
         except:
             self.source_handle = None
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
+        object = self.get_object(data)
         if not self.source_handle:
             if self.nosource:
                 # check whether the citation list is empty as a proxy for
diff --git a/gramps/gen/filters/rules/_hastagbase.py b/gramps/gen/filters/rules/_hastagbase.py
index 0451dba66a..edc9fbaeeb 100644
--- a/gramps/gen/filters/rules/_hastagbase.py
+++ b/gramps/gen/filters/rules/_hastagbase.py
@@ -70,9 +70,6 @@ def prepare(self, db, user):
         else:
             self.map = set()
 
-    def get_rules_with_maps(self):
-        return [self]
-
     def apply_to_one(self, db, data):
         """
         Apply the rule.  Return True for a match.
diff --git a/gramps/gen/filters/rules/_isprivate.py b/gramps/gen/filters/rules/_isprivate.py
index 53c2b95a0f..bd3e804d2c 100644
--- a/gramps/gen/filters/rules/_isprivate.py
+++ b/gramps/gen/filters/rules/_isprivate.py
@@ -43,5 +43,5 @@ class IsPrivate(Rule):
     description = "Matches objects that are indicated as private"
     category = _("General filters")
 
-    def apply(self, db, obj):
-        return obj.get_privacy()
+    def apply_to_one(self, db, data):
+        return data["private"]
diff --git a/gramps/gen/filters/rules/_ispublic.py b/gramps/gen/filters/rules/_ispublic.py
index 2ffdf60812..180354b743 100644
--- a/gramps/gen/filters/rules/_ispublic.py
+++ b/gramps/gen/filters/rules/_ispublic.py
@@ -40,5 +40,5 @@ class IsPublic(Rule):
     description = "Matches objects that are not indicated as private"
     category = _("General filters")
 
-    def apply(self, db, obj):
-        return not obj.get_privacy()
+    def apply_to_one(self, db, data):
+        return not data["private"]
diff --git a/gramps/gen/filters/rules/_matcheseventfilterbase.py b/gramps/gen/filters/rules/_matcheseventfilterbase.py
index 9e551392f3..4f8655df7d 100644
--- a/gramps/gen/filters/rules/_matcheseventfilterbase.py
+++ b/gramps/gen/filters/rules/_matcheseventfilterbase.py
@@ -61,13 +61,14 @@ def prepare(self, db, user):
         MatchesFilterBase.prepare(self, db, user)
         self.MEF_filt = self.find_filter()
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
+        object = self.get_object(data)
         if self.MEF_filt is None:
             return False
 
         eventlist = [x.ref for x in object.get_event_ref_list()]
         for eventhandle in eventlist:
             # check if event in event filter
-            if self.MEF_filt.check(db, eventhandle):
+            if self.MEF_filt.apply_to_one(db, eventhandle):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/_matchesfilterbase.py b/gramps/gen/filters/rules/_matchesfilterbase.py
index df979a47fb..71dd6c29cd 100644
--- a/gramps/gen/filters/rules/_matchesfilterbase.py
+++ b/gramps/gen/filters/rules/_matchesfilterbase.py
@@ -90,7 +90,6 @@ def apply_to_one(self, db, data):
             filters = gramps.gen.filters.CustomFilters.get_filters_dict(self.namespace)
             if self.list[0] in filters:
                 filt = filters[self.list[0]]
-                # FIXME: Is this correct?
                 return filt.apply_to_one(db, data)
         return False
 
diff --git a/gramps/gen/filters/rules/_matchessourceconfidencebase.py b/gramps/gen/filters/rules/_matchessourceconfidencebase.py
index ccb9e8eb05..f871c289d3 100644
--- a/gramps/gen/filters/rules/_matchessourceconfidencebase.py
+++ b/gramps/gen/filters/rules/_matchessourceconfidencebase.py
@@ -53,7 +53,8 @@ class MatchesSourceConfidenceBase(Rule):
     )
     category = _("Citation/source filters")
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
+        obj = self.get_object(data)
         required_conf = int(self.list[0])
         for citation_handle in obj.get_citation_list():
             citation = db.get_citation_from_handle(citation_handle)
diff --git a/gramps/gen/filters/rules/_matchessourcefilterbase.py b/gramps/gen/filters/rules/_matchessourcefilterbase.py
index 1f9e980174..a471e20328 100644
--- a/gramps/gen/filters/rules/_matchessourcefilterbase.py
+++ b/gramps/gen/filters/rules/_matchessourcefilterbase.py
@@ -60,13 +60,15 @@ def prepare(self, db, user):
         MatchesFilterBase.prepare(self, db, user)
         self.MSF_filt = self.find_filter()
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
         if self.MSF_filt is None:
             return False
 
+        object = self.get_object(data)
         for citation_handle in object.get_citation_list():
             citation = db.get_citation_from_handle(citation_handle)
             sourcehandle = citation.get_reference_handle()
-            if self.MSF_filt.check(db, sourcehandle):
+            source_data = db.get_raw_source_data(sourcehandle)
+            if self.MSF_filt.apply_to_one(db, source_data):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/_regexpidbase.py b/gramps/gen/filters/rules/_regexpidbase.py
index 14b5c9a687..e0ffad2fe1 100644
--- a/gramps/gen/filters/rules/_regexpidbase.py
+++ b/gramps/gen/filters/rules/_regexpidbase.py
@@ -56,5 +56,5 @@ class RegExpIdBase(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, obj):
-        return self.match_substring(0, obj.gramps_id)
+    def apply_to_one(self, db, data):
+        return self.match_substring(0, data["gramps_id"])
diff --git a/gramps/gen/filters/rules/_rule.py b/gramps/gen/filters/rules/_rule.py
index 544b5c5530..610f5b96ff 100644
--- a/gramps/gen/filters/rules/_rule.py
+++ b/gramps/gen/filters/rules/_rule.py
@@ -32,6 +32,7 @@
 
 from ...errors import FilterError
 from ...const import GRAMPS_LOCALE as glocale
+from ...lib.serialize import from_dict
 
 _ = glocale.translation.gettext
 
@@ -154,6 +155,14 @@ def apply_to_one(self, dummy_db, dummy_data):
         """Apply the rule to some database entry; must be overwritten."""
         return True
 
+    def get_object(self, data):
+        """
+        Create an object, but only do it once per data.
+        """
+        if "_object" not in data:
+            data["_object"] = from_dict(data)
+        return data["_object"]
+
     def display_values(self):
         """Return the labels and values of this rule."""
         l_v = (
diff --git a/gramps/gen/filters/rules/citation/_hascitation.py b/gramps/gen/filters/rules/citation/_hascitation.py
index af08fc5a40..635a51ca9e 100644
--- a/gramps/gen/filters/rules/citation/_hascitation.py
+++ b/gramps/gen/filters/rules/citation/_hascitation.py
@@ -61,7 +61,8 @@ def prepare(self, db, user):
         except:
             pass
 
-    def apply(self, dbase, citation):
+    def apply_to_one(self, dbase, data):
+        citation = self.get_object(data)
         if not self.match_substring(0, citation.get_page()):
             return False
 
diff --git a/gramps/gen/filters/rules/citation/_hassource.py b/gramps/gen/filters/rules/citation/_hassource.py
index 1a63bfcaaf..32d545092c 100644
--- a/gramps/gen/filters/rules/citation/_hassource.py
+++ b/gramps/gen/filters/rules/citation/_hassource.py
@@ -52,8 +52,9 @@ class HasSource(HasSourceBase):
     description = _("Matches citations with a source of a particular " "value")
     category = _("Source filters")
 
-    def apply(self, dbase, citation):
+    def apply_to_one(self, dbase, data):
+        citation = self.get_object(data)
         source = dbase.get_source_from_handle(citation.get_reference_handle())
-        if HasSourceBase.apply(self, dbase, source):
+        if HasSourceBase.apply_to_one(self, dbase, source.handle):
             return True
         return False
diff --git a/gramps/gen/filters/rules/citation/_hassourceidof.py b/gramps/gen/filters/rules/citation/_hassourceidof.py
index de6e07b765..095ab5a32a 100644
--- a/gramps/gen/filters/rules/citation/_hassourceidof.py
+++ b/gramps/gen/filters/rules/citation/_hassourceidof.py
@@ -34,6 +34,7 @@
 #
 # -------------------------------------------------------------------------
 from .._hasgrampsid import HasGrampsId
+from gramps.gen.lib.serialize import to_dict
 
 
 # -------------------------------------------------------------------------
@@ -49,8 +50,9 @@ class HasSourceIdOf(HasGrampsId):
     description = _("Matches a citation with a source with a specified Gramps " "ID")
     category = _("Source filters")
 
-    def apply(self, dbase, citation):
+    def apply_to_one(self, dbase, data):
+        citation = self.get_object(data)
         source = dbase.get_source_from_handle(citation.get_reference_handle())
-        if HasGrampsId.apply(self, dbase, source):
+        if HasGrampsId.apply_to_one(self, dbase, to_dict(source)):
             return True
         return False
diff --git a/gramps/gen/filters/rules/citation/_hassourcenoteregexp.py b/gramps/gen/filters/rules/citation/_hassourcenoteregexp.py
index 82ceeb2d4e..dcd0bdeae5 100644
--- a/gramps/gen/filters/rules/citation/_hassourcenoteregexp.py
+++ b/gramps/gen/filters/rules/citation/_hassourcenoteregexp.py
@@ -38,6 +38,7 @@
 #
 # -------------------------------------------------------------------------
 from .._hasnoteregexbase import HasNoteRegexBase
+from gramps.gen.lib.serialize import to_dict
 
 
 # -------------------------------------------------------------------------
@@ -58,8 +59,9 @@ class HasSourceNoteRegexp(HasNoteRegexBase):
     )
     category = _("Source filters")
 
-    def apply(self, db, citation):
+    def apply_to_one(self, db, data):
+        citation = self.get_object(data)
         source = db.get_source_from_handle(citation.get_reference_handle())
-        if HasNoteRegexBase.apply(self, db, source):
+        if HasNoteRegexBase.apply_to_one(self, db, to_dict(source)):
             return True
         return False
diff --git a/gramps/gen/filters/rules/citation/_hastag.py b/gramps/gen/filters/rules/citation/_hastag.py
index efe1a330fb..9074417eb9 100644
--- a/gramps/gen/filters/rules/citation/_hastag.py
+++ b/gramps/gen/filters/rules/citation/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Citations with the <tag>")
     description = _("Matches citations with the particular tag")
+    table = "citation"
diff --git a/gramps/gen/filters/rules/citation/_matchespagesubstringof.py b/gramps/gen/filters/rules/citation/_matchespagesubstringof.py
index f45abf4650..ec49df97b8 100644
--- a/gramps/gen/filters/rules/citation/_matchespagesubstringof.py
+++ b/gramps/gen/filters/rules/citation/_matchespagesubstringof.py
@@ -49,6 +49,6 @@ class MatchesPageSubstringOf(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
         """Apply the filter"""
-        return self.match_substring(0, object.get_page())
+        return self.match_substring(0, data["page"])
diff --git a/gramps/gen/filters/rules/citation/_matchesrepositoryfilter.py b/gramps/gen/filters/rules/citation/_matchesrepositoryfilter.py
index a9260647d4..2e2a7a8e75 100644
--- a/gramps/gen/filters/rules/citation/_matchesrepositoryfilter.py
+++ b/gramps/gen/filters/rules/citation/_matchesrepositoryfilter.py
@@ -63,7 +63,8 @@ def prepare(self, db, user):
         MatchesFilterBase.prepare(self, db, user)
         self.MRF_filt = self.find_filter()
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
+        object = self.get_object(data)
         if self.MRF_filt is None:
             return False
 
diff --git a/gramps/gen/filters/rules/citation/_matchessourcefilter.py b/gramps/gen/filters/rules/citation/_matchessourcefilter.py
index 0bc9c18fc2..bbbcf8b35f 100644
--- a/gramps/gen/filters/rules/citation/_matchessourcefilter.py
+++ b/gramps/gen/filters/rules/citation/_matchessourcefilter.py
@@ -61,11 +61,11 @@ def prepare(self, db, user):
         MatchesFilterBase.prepare(self, db, user)
         self.MRF_filt = self.find_filter()
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
         if self.MRF_filt is None:
             return False
 
-        source_handle = object.source_handle
-        if self.MRF_filt.check(db, source_handle):
+        source_handle = data["source_handle"]
+        if self.MRF_filt.apply_to_one(db, source_handle):
             return True
         return False
diff --git a/gramps/gen/filters/rules/citation/_regexpsourceidof.py b/gramps/gen/filters/rules/citation/_regexpsourceidof.py
index afa7f2471a..57cd68dc28 100644
--- a/gramps/gen/filters/rules/citation/_regexpsourceidof.py
+++ b/gramps/gen/filters/rules/citation/_regexpsourceidof.py
@@ -34,6 +34,7 @@
 #
 # -------------------------------------------------------------------------
 from .._regexpidbase import RegExpIdBase
+from gramps.gen.lib.serialize import to_dict
 
 
 # -------------------------------------------------------------------------
@@ -54,8 +55,9 @@ class RegExpSourceIdOf(RegExpIdBase):
     )
     category = _("Source filters")
 
-    def apply(self, dbase, citation):
+    def apply_to_one(self, dbase, data):
+        citation = self.get_object(data)
         source = dbase.get_source_from_handle(citation.get_reference_handle())
-        if RegExpIdBase.apply(self, dbase, source):
+        if RegExpIdBase.apply_to_one(self, dbase, to_dict(source)):
             return True
         return False
diff --git a/gramps/gen/filters/rules/event/_hasdata.py b/gramps/gen/filters/rules/event/_hasdata.py
index 51352930af..ea183c4778 100644
--- a/gramps/gen/filters/rules/event/_hasdata.py
+++ b/gramps/gen/filters/rules/event/_hasdata.py
@@ -71,10 +71,11 @@ def prepare(self, db, user):
         if self.date:
             self.date = parser.parse(self.date)
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         if self.event_type and obj.get_type() != self.event_type:
             # No match
             return False
diff --git a/gramps/gen/filters/rules/event/_hasdayofweek.py b/gramps/gen/filters/rules/event/_hasdayofweek.py
index b4aad3c21e..82169cd0ce 100644
--- a/gramps/gen/filters/rules/event/_hasdayofweek.py
+++ b/gramps/gen/filters/rules/event/_hasdayofweek.py
@@ -42,7 +42,8 @@ class HasDayOfWeek(Rule):
     description = _("Matches events occurring on a particular day of the week")
     category = _("General filters")
 
-    def apply(self, db, event):
+    def apply_to_one(self, db, data):
+        event = self.get_object(data)
         if not self.list[0]:
             return False
         else:
diff --git a/gramps/gen/filters/rules/event/_hastag.py b/gramps/gen/filters/rules/event/_hastag.py
index b2e0ef303c..df151c71d5 100644
--- a/gramps/gen/filters/rules/event/_hastag.py
+++ b/gramps/gen/filters/rules/event/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Events with the <tag>")
     description = _("Matches events with the particular tag")
+    table = "event"
diff --git a/gramps/gen/filters/rules/event/_hastype.py b/gramps/gen/filters/rules/event/_hastype.py
index bec57745b7..5e08352fff 100644
--- a/gramps/gen/filters/rules/event/_hastype.py
+++ b/gramps/gen/filters/rules/event/_hastype.py
@@ -61,10 +61,12 @@ def prepare(self, db, user):
             self.event_type = EventType()
             self.event_type.set_from_xml_str(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True if a match.
         """
-        if self.event_type:
+        if data["event_type"]:
+            obj = self.get_object(data)
+            # FIXME: probably don't need object to do this:
             return obj.get_type() == self.event_type
         return False
diff --git a/gramps/gen/filters/rules/event/_matchespersonfilter.py b/gramps/gen/filters/rules/event/_matchespersonfilter.py
index b82cb22e33..deec82fe12 100644
--- a/gramps/gen/filters/rules/event/_matchespersonfilter.py
+++ b/gramps/gen/filters/rules/event/_matchespersonfilter.py
@@ -70,23 +70,32 @@ def prepare(self, db, user):
         except IndexError:
             self.MPF_famevents = False
 
-    def apply(self, db, event):
+    def apply_to_one(self, db, event_data):
         filt = self.find_filter()
         if filt:
             for classname, handle in db.find_backlink_handles(
-                event.get_handle(), ["Person"]
+                event_data["handle"], ["Person"]
             ):
-                if filt.check(db, handle):
+                data = db.get_raw_person_data(handle)
+                if filt.apply_to_one(db, data):
                     return True
             if self.MPF_famevents:
                 # also include if family event of the person
                 for classname, handle in db.find_backlink_handles(
-                    event.get_handle(), ["Family"]
+                    event_data["handle"], ["Family"]
                 ):
-                    family = db.get_family_from_handle(handle)
-                    if family.father_handle and filt.check(db, family.father_handle):
-                        return True
-                    if family.mother_handle and filt.check(db, family.mother_handle):
-                        return True
+                    family_data = db.get_raw_family_data(handle)
+                    if family_data["father_handle"]:
+                        father_data = db.get_raw_person_data(
+                            family_data["father_handle"]
+                        )
+                        if filt.apply_to_one(db, father_data):
+                            return True
+                    if family_data["mother_handle"]:
+                        mother_data = db.get_raw_person_data(
+                            family_data["mother_handle"]
+                        )
+                        if filt.apply_to_one(db, mother_data):
+                            return True
 
         return False
diff --git a/gramps/gen/filters/rules/event/_matchesplacefilter.py b/gramps/gen/filters/rules/event/_matchesplacefilter.py
index b05924b0fd..2a2614f0c1 100644
--- a/gramps/gen/filters/rules/event/_matchesplacefilter.py
+++ b/gramps/gen/filters/rules/event/_matchesplacefilter.py
@@ -33,6 +33,7 @@
 #
 # -------------------------------------------------------------------------
 from .._matchesfilterbase import MatchesFilterBase
+from gramps.gen.lib.serialize import to_dict
 
 
 # -------------------------------------------------------------------------
@@ -58,10 +59,11 @@ class MatchesPlaceFilter(MatchesFilterBase):
     # we want to have this filter show place filters
     namespace = "Place"
 
-    def apply(self, db, event):
+    def apply_to_one(self, db, data):
+        event = self.get_object(data)
         filt = self.find_filter()
         if filt:
             handle = event.get_place_handle()
-            if handle and filt.check(db, handle):
+            if handle and filt.apply_to_one(db, to_dict(event)):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/family/_childhasidof.py b/gramps/gen/filters/rules/family/_childhasidof.py
index ab1b206924..4c8a670dfb 100644
--- a/gramps/gen/filters/rules/family/_childhasidof.py
+++ b/gramps/gen/filters/rules/family/_childhasidof.py
@@ -49,4 +49,4 @@ class ChildHasIdOf(RegExpIdBase):
     description = _("Matches families where child has a specified " "Gramps ID")
     category = _("Child filters")
     base_class = RegExpIdBase
-    apply = child_base
+    apply_to_one = child_base
diff --git a/gramps/gen/filters/rules/family/_childhasnameof.py b/gramps/gen/filters/rules/family/_childhasnameof.py
index 0da60f5488..1ad64b62d2 100644
--- a/gramps/gen/filters/rules/family/_childhasnameof.py
+++ b/gramps/gen/filters/rules/family/_childhasnameof.py
@@ -48,4 +48,4 @@ class ChildHasNameOf(HasNameOf):
     description = _("Matches families where child has a specified " "(partial) name")
     category = _("Child filters")
     base_class = HasNameOf
-    apply = child_base
+    apply_to_one = child_base
diff --git a/gramps/gen/filters/rules/family/_fatherhasidof.py b/gramps/gen/filters/rules/family/_fatherhasidof.py
index 5b8ccf79bb..f3754d1c44 100644
--- a/gramps/gen/filters/rules/family/_fatherhasidof.py
+++ b/gramps/gen/filters/rules/family/_fatherhasidof.py
@@ -49,4 +49,4 @@ class FatherHasIdOf(RegExpIdBase):
     description = _("Matches families whose father has a specified " "Gramps ID")
     category = _("Father filters")
     base_class = RegExpIdBase
-    apply = father_base
+    apply_to_one = father_base
diff --git a/gramps/gen/filters/rules/family/_fatherhasnameof.py b/gramps/gen/filters/rules/family/_fatherhasnameof.py
index c9030f313f..ae0d6294c3 100644
--- a/gramps/gen/filters/rules/family/_fatherhasnameof.py
+++ b/gramps/gen/filters/rules/family/_fatherhasnameof.py
@@ -48,4 +48,4 @@ class FatherHasNameOf(HasNameOf):
     description = _("Matches families whose father has a specified " "(partial) name")
     category = _("Father filters")
     base_class = HasNameOf
-    apply = father_base
+    apply_to_one = father_base
diff --git a/gramps/gen/filters/rules/family/_hasevent.py b/gramps/gen/filters/rules/family/_hasevent.py
index 0662e54670..43abf83d00 100644
--- a/gramps/gen/filters/rules/family/_hasevent.py
+++ b/gramps/gen/filters/rules/family/_hasevent.py
@@ -56,7 +56,8 @@ class HasEvent(HasEventBase):
     name = _("Families with the <event>")
     description = _("Matches families with an event of a particular value")
 
-    def apply(self, dbase, family):
+    def apply_to_one(self, dbase, data):
+        family = self.get_object(data)
         for event_ref in family.get_event_ref_list():
             if not event_ref:
                 continue
diff --git a/gramps/gen/filters/rules/family/_hasnote.py b/gramps/gen/filters/rules/family/_hasnote.py
index 5a3c981cd1..b742148c6b 100644
--- a/gramps/gen/filters/rules/family/_hasnote.py
+++ b/gramps/gen/filters/rules/family/_hasnote.py
@@ -46,3 +46,4 @@ class HasNote(HasNoteBase):
 
     name = _("Families having <count> notes")
     description = _("Matches families having a certain number notes")
+    table = "family"
diff --git a/gramps/gen/filters/rules/family/_hasreltype.py b/gramps/gen/filters/rules/family/_hasreltype.py
index 6b59e88b5d..3844d8f7ca 100644
--- a/gramps/gen/filters/rules/family/_hasreltype.py
+++ b/gramps/gen/filters/rules/family/_hasreltype.py
@@ -61,10 +61,11 @@ def prepare(self, db, user):
             self.relation_type = FamilyRelType()
             self.relation_type.set_from_xml_str(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         if self.relation_type:
             if self.relation_type.is_custom() and self.use_regex:
                 if self.regex[0].search(str(obj.get_relationship())) is None:
diff --git a/gramps/gen/filters/rules/family/_hastag.py b/gramps/gen/filters/rules/family/_hastag.py
index e559fac212..617368881b 100644
--- a/gramps/gen/filters/rules/family/_hastag.py
+++ b/gramps/gen/filters/rules/family/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Families with the <tag>")
     description = _("Matches families with the particular tag")
+    table = "family"
diff --git a/gramps/gen/filters/rules/family/_hastwins.py b/gramps/gen/filters/rules/family/_hastwins.py
index fdaa8f1964..cb0ca151ee 100644
--- a/gramps/gen/filters/rules/family/_hastwins.py
+++ b/gramps/gen/filters/rules/family/_hastwins.py
@@ -48,7 +48,8 @@ class HasTwins(Rule):
     description = _("Matches families with twins")
     category = _("Child filters")
 
-    def apply(self, db, family):
+    def apply_to_one(self, db, data):
+        family = self.get_object(data)
         date_list = []
         for childref in family.get_child_ref_list():
             if int(childref.get_mother_relation()) == ChildRefType.BIRTH:
diff --git a/gramps/gen/filters/rules/family/_isancestorof.py b/gramps/gen/filters/rules/family/_isancestorof.py
index f9d9473b7e..9b1f9efb75 100644
--- a/gramps/gen/filters/rules/family/_isancestorof.py
+++ b/gramps/gen/filters/rules/family/_isancestorof.py
@@ -58,8 +58,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, family):
-        return family.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, db, family, first):
         """
diff --git a/gramps/gen/filters/rules/family/_isbookmarked.py b/gramps/gen/filters/rules/family/_isbookmarked.py
index d3cfdf40c2..0ca803a4ba 100644
--- a/gramps/gen/filters/rules/family/_isbookmarked.py
+++ b/gramps/gen/filters/rules/family/_isbookmarked.py
@@ -50,5 +50,5 @@ class IsBookmarked(Rule):
     def prepare(self, db, user):
         self.bookmarks = db.get_family_bookmarks().get()
 
-    def apply(self, db, family):
-        return family.get_handle() in self.bookmarks
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.bookmarks
diff --git a/gramps/gen/filters/rules/family/_isdescendantof.py b/gramps/gen/filters/rules/family/_isdescendantof.py
index 52b0bc48ac..321c202256 100644
--- a/gramps/gen/filters/rules/family/_isdescendantof.py
+++ b/gramps/gen/filters/rules/family/_isdescendantof.py
@@ -58,8 +58,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, family):
-        return family.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, db, family, first):
         """
diff --git a/gramps/gen/filters/rules/family/_memberbase.py b/gramps/gen/filters/rules/family/_memberbase.py
index a042484ca2..086d3047bd 100644
--- a/gramps/gen/filters/rules/family/_memberbase.py
+++ b/gramps/gen/filters/rules/family/_memberbase.py
@@ -32,7 +32,8 @@
 """
 
 
-def father_base(self, db, family):
+def father_base(self, db, data):
+    family = self.get_object(data)
     father_handle = family.get_father_handle()
     if father_handle:
         father = db.get_person_from_handle(father_handle)
@@ -42,7 +43,8 @@ def father_base(self, db, family):
             return False
 
 
-def mother_base(self, db, family):
+def mother_base(self, db, data):
+    family = self.get_object(data)
     mother_handle = family.get_mother_handle()
     if mother_handle:
         mother = db.get_person_from_handle(mother_handle)
@@ -52,7 +54,8 @@ def mother_base(self, db, family):
             return False
 
 
-def child_base(self, db, family):
+def child_base(self, db, data):
+    family = self.get_object(data)
     for child_ref in family.get_child_ref_list():
         child = db.get_person_from_handle(child_ref.ref)
         if self.base_class.apply(self, db, child):
diff --git a/gramps/gen/filters/rules/family/_motherhasidof.py b/gramps/gen/filters/rules/family/_motherhasidof.py
index 0c8ea2f6ed..bababa6756 100644
--- a/gramps/gen/filters/rules/family/_motherhasidof.py
+++ b/gramps/gen/filters/rules/family/_motherhasidof.py
@@ -49,4 +49,4 @@ class MotherHasIdOf(RegExpIdBase):
     description = _("Matches families whose mother has a specified " "Gramps ID")
     category = _("Mother filters")
     base_class = RegExpIdBase
-    apply = mother_base
+    apply_to_one = mother_base
diff --git a/gramps/gen/filters/rules/family/_motherhasnameof.py b/gramps/gen/filters/rules/family/_motherhasnameof.py
index 1872cd46c3..51288c8212 100644
--- a/gramps/gen/filters/rules/family/_motherhasnameof.py
+++ b/gramps/gen/filters/rules/family/_motherhasnameof.py
@@ -48,4 +48,4 @@ class MotherHasNameOf(HasNameOf):
     description = _("Matches families whose mother has a specified " "(partial) name")
     category = _("Mother filters")
     base_class = HasNameOf
-    apply = mother_base
+    apply_to_one = mother_base
diff --git a/gramps/gen/filters/rules/family/_regexpchildname.py b/gramps/gen/filters/rules/family/_regexpchildname.py
index f0fb746b1c..4813e04e5f 100644
--- a/gramps/gen/filters/rules/family/_regexpchildname.py
+++ b/gramps/gen/filters/rules/family/_regexpchildname.py
@@ -51,4 +51,4 @@ class RegExpChildName(RegExpName):
     )
     category = _("Child filters")
     base_class = RegExpName
-    apply = child_base
+    apply_to_one = child_base
diff --git a/gramps/gen/filters/rules/family/_regexpfathername.py b/gramps/gen/filters/rules/family/_regexpfathername.py
index 6fe0e9ddde..81ac8f6d77 100644
--- a/gramps/gen/filters/rules/family/_regexpfathername.py
+++ b/gramps/gen/filters/rules/family/_regexpfathername.py
@@ -51,4 +51,4 @@ class RegExpFatherName(RegExpName):
     )
     category = _("Father filters")
     base_class = RegExpName
-    apply = father_base
+    apply_to_one = father_base
diff --git a/gramps/gen/filters/rules/family/_regexpmothername.py b/gramps/gen/filters/rules/family/_regexpmothername.py
index 24378f1075..f6f068994c 100644
--- a/gramps/gen/filters/rules/family/_regexpmothername.py
+++ b/gramps/gen/filters/rules/family/_regexpmothername.py
@@ -51,4 +51,4 @@ class RegExpMotherName(RegExpName):
     )
     category = _("Mother filters")
     base_class = RegExpName
-    apply = mother_base
+    apply_to_one = mother_base
diff --git a/gramps/gen/filters/rules/family/_searchchildname.py b/gramps/gen/filters/rules/family/_searchchildname.py
index 2450430a28..1fbe1b4dc4 100644
--- a/gramps/gen/filters/rules/family/_searchchildname.py
+++ b/gramps/gen/filters/rules/family/_searchchildname.py
@@ -50,4 +50,4 @@ class SearchChildName(SearchName):
     )
     category = _("Child filters")
     base_class = SearchName
-    apply = child_base
+    apply_to_one = child_base
diff --git a/gramps/gen/filters/rules/family/_searchfathername.py b/gramps/gen/filters/rules/family/_searchfathername.py
index 431717d8cb..741032fa0d 100644
--- a/gramps/gen/filters/rules/family/_searchfathername.py
+++ b/gramps/gen/filters/rules/family/_searchfathername.py
@@ -48,4 +48,4 @@ class SearchFatherName(SearchName):
     description = _("Matches families whose father has a specified " "(partial) name")
     category = _("Father filters")
     base_class = SearchName
-    apply = father_base
+    apply_to_one = father_base
diff --git a/gramps/gen/filters/rules/family/_searchmothername.py b/gramps/gen/filters/rules/family/_searchmothername.py
index 8bbca71048..e91458aeb5 100644
--- a/gramps/gen/filters/rules/family/_searchmothername.py
+++ b/gramps/gen/filters/rules/family/_searchmothername.py
@@ -48,4 +48,4 @@ class SearchMotherName(SearchName):
     description = _("Matches families whose mother has a specified " "(partial) name")
     category = _("Mother filters")
     base_class = SearchName
-    apply = mother_base
+    apply_to_one = mother_base
diff --git a/gramps/gen/filters/rules/media/_hasmedia.py b/gramps/gen/filters/rules/media/_hasmedia.py
index 49ffc7f397..5e0d53beea 100644
--- a/gramps/gen/filters/rules/media/_hasmedia.py
+++ b/gramps/gen/filters/rules/media/_hasmedia.py
@@ -63,7 +63,8 @@ def prepare(self, db, user):
         except:
             pass
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
+        obj = self.get_object(data)
         if not self.match_substring(0, obj.get_description()):
             return False
 
diff --git a/gramps/gen/filters/rules/note/_hasnote.py b/gramps/gen/filters/rules/note/_hasnote.py
index d9ed302d0a..52b99bf1ba 100644
--- a/gramps/gen/filters/rules/note/_hasnote.py
+++ b/gramps/gen/filters/rules/note/_hasnote.py
@@ -65,10 +65,11 @@ def prepare(self, db, user):
             self.note_type = NoteType()
             self.note_type.set_from_xml_str(self.list[1])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         if not self.match_substring(0, obj.get()):
             return False
 
diff --git a/gramps/gen/filters/rules/note/_hastype.py b/gramps/gen/filters/rules/note/_hastype.py
index d81f522b8d..98f3b440e1 100644
--- a/gramps/gen/filters/rules/note/_hastype.py
+++ b/gramps/gen/filters/rules/note/_hastype.py
@@ -61,10 +61,11 @@ def prepare(self, db, user):
             self.note_type = NoteType()
             self.note_type.set_from_xml_str(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         if self.note_type:
             return obj.get_type() == self.note_type
         return False
diff --git a/gramps/gen/filters/rules/note/_matchesregexpof.py b/gramps/gen/filters/rules/note/_matchesregexpof.py
index ad8c5ada03..95eabccc61 100644
--- a/gramps/gen/filters/rules/note/_matchesregexpof.py
+++ b/gramps/gen/filters/rules/note/_matchesregexpof.py
@@ -49,8 +49,9 @@ class MatchesRegexpOf(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, note):
+    def apply_to_one(self, db, data):
         """Apply the filter"""
+        note = self.get_object(data)
         if self.match_substring(0, note.get()):
             return True
         return False
diff --git a/gramps/gen/filters/rules/note/_matchessubstringof.py b/gramps/gen/filters/rules/note/_matchessubstringof.py
index 96cafb9fb0..b6ece36d1e 100644
--- a/gramps/gen/filters/rules/note/_matchessubstringof.py
+++ b/gramps/gen/filters/rules/note/_matchessubstringof.py
@@ -47,8 +47,9 @@ class MatchesSubstringOf(Rule):
     description = _("Matches notes that contain text " "which matches a substring")
     category = _("General filters")
 
-    def apply(self, db, note):
+    def apply_to_one(self, db, data):
         """Apply the filter"""
+        note = self.get_object(data)
         text = note.get()
         if text.upper().find(self.list[0].upper()) != -1:
             return True
diff --git a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
index d617b6e36f..eed5415be0 100644
--- a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
+++ b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
@@ -170,13 +170,13 @@ def prepare(self, db, user):
                 _("Evaluating people"),
                 db.get_number_of_people(),
             )
-        self.__matches = find_deep_relations(db, user, root_person, target_people)
+        self.map = find_deep_relations(db, user, root_person, target_people)
         if user:
             user.end_progress()
 
     def reset(self):
         self.filt.requestreset()
-        self.__matches = set()
+        self.map.clear()
 
-    def apply(self, db, person):
-        return person.get_handle() in self.__matches
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.__matches
diff --git a/gramps/gen/filters/rules/person/_disconnected.py b/gramps/gen/filters/rules/person/_disconnected.py
index 66b60c2876..8df8f84a91 100644
--- a/gramps/gen/filters/rules/person/_disconnected.py
+++ b/gramps/gen/filters/rules/person/_disconnected.py
@@ -50,7 +50,5 @@ class Disconnected(Rule):
         "to any other person in the database"
     )
 
-    def apply(self, db, person):
-        return not (
-            person.get_parent_family_handle_list() or person.get_family_handle_list()
-        )
+    def apply_to_one(self, db, data):
+        return not (data["parent_family_list"] or data["family_list"])
diff --git a/gramps/gen/filters/rules/person/_everyone.py b/gramps/gen/filters/rules/person/_everyone.py
index 29832d2d67..48695e40cf 100644
--- a/gramps/gen/filters/rules/person/_everyone.py
+++ b/gramps/gen/filters/rules/person/_everyone.py
@@ -50,5 +50,5 @@ class Everyone(Rule):
     def is_empty(self):
         return True
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
         return True
diff --git a/gramps/gen/filters/rules/person/_familywithincompleteevent.py b/gramps/gen/filters/rules/person/_familywithincompleteevent.py
index c5ff2bdb3f..4952049fa7 100644
--- a/gramps/gen/filters/rules/person/_familywithincompleteevent.py
+++ b/gramps/gen/filters/rules/person/_familywithincompleteevent.py
@@ -47,7 +47,8 @@ class FamilyWithIncompleteEvent(Rule):
     )
     category = _("Event filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for family_handle in person.get_family_handle_list():
             family = db.get_family_from_handle(family_handle)
             if family:
diff --git a/gramps/gen/filters/rules/person/_hasaddress.py b/gramps/gen/filters/rules/person/_hasaddress.py
index 6c8b891aec..5197d96db7 100644
--- a/gramps/gen/filters/rules/person/_hasaddress.py
+++ b/gramps/gen/filters/rules/person/_hasaddress.py
@@ -63,8 +63,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, person):
-        count = len(person.get_address_list())
+    def apply_to_one(self, db, data):
+        count = len(data["address_list"])
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
         elif self.count_type == 2:  # "greater than"
diff --git a/gramps/gen/filters/rules/person/_hasaddresstext.py b/gramps/gen/filters/rules/person/_hasaddresstext.py
index a42b973634..6c63dabac8 100644
--- a/gramps/gen/filters/rules/person/_hasaddresstext.py
+++ b/gramps/gen/filters/rules/person/_hasaddresstext.py
@@ -48,7 +48,8 @@ class HasAddressText(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for address in person.get_address_list():
             for string in address.get_text_data_list():
                 if self.match_substring(0, string):
diff --git a/gramps/gen/filters/rules/person/_hasalternatename.py b/gramps/gen/filters/rules/person/_hasalternatename.py
index 673111879e..0c0485c9a2 100644
--- a/gramps/gen/filters/rules/person/_hasalternatename.py
+++ b/gramps/gen/filters/rules/person/_hasalternatename.py
@@ -48,8 +48,8 @@ class HasAlternateName(Rule):
     description = _("Matches people with an alternate name")
     category = _("General filters")
 
-    def apply(self, db, person):
-        if person.get_alternate_names():
+    def apply_to_one(self, db, data):
+        if data["alternate_names"]:
             return True
         else:
             return False
diff --git a/gramps/gen/filters/rules/person/_hasassociation.py b/gramps/gen/filters/rules/person/_hasassociation.py
index e1d95f8c31..9010445c07 100644
--- a/gramps/gen/filters/rules/person/_hasassociation.py
+++ b/gramps/gen/filters/rules/person/_hasassociation.py
@@ -63,8 +63,8 @@ def prepare(self, db, user):
 
         self.selected_count = int(self.list[0])
 
-    def apply(self, db, person):
-        count = len(person.get_person_ref_list())
+    def apply_to_one(self, db, data):
+        count = len(data["person_ref_list"])
         if self.count_type == 0:  # "less than"
             return count < self.selected_count
         elif self.count_type == 2:  # "greater than"
diff --git a/gramps/gen/filters/rules/person/_hasbirth.py b/gramps/gen/filters/rules/person/_hasbirth.py
index 511a61fa82..74035adc33 100644
--- a/gramps/gen/filters/rules/person/_hasbirth.py
+++ b/gramps/gen/filters/rules/person/_hasbirth.py
@@ -59,7 +59,8 @@ def prepare(self, db, user):
         else:
             self.date = None
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for event_ref in person.get_event_ref_list():
             if not event_ref:
                 continue
diff --git a/gramps/gen/filters/rules/person/_hascommonancestorwith.py b/gramps/gen/filters/rules/person/_hascommonancestorwith.py
index cba92f5df1..5b90780b12 100644
--- a/gramps/gen/filters/rules/person/_hascommonancestorwith.py
+++ b/gramps/gen/filters/rules/person/_hascommonancestorwith.py
@@ -106,8 +106,9 @@ def has_common_ancestor(self, other):
                 return True
         return False
 
-    def apply(self, db, person):
-        if person and person.handle not in self.ancestor_cache:
+    def apply_to_one(self, db, data):
+        if data["handle"] not in self.ancestor_cache:
+            person = self.get_object(data)
             self.add_ancs(db, person)
 
         return self.has_common_ancestor(person)
diff --git a/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py b/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
index 78183cfa99..7aa4c49725 100644
--- a/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
+++ b/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
@@ -35,6 +35,7 @@
 from ....utils.db import for_each_ancestor
 from ._hascommonancestorwith import HasCommonAncestorWith
 from ._matchesfilter import MatchesFilter
+from gramps.gen.lib.serialize import from_dict
 
 
 # -------------------------------------------------------------------------
@@ -73,15 +74,16 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for handle in db.iter_person_handles():
-            person = db.get_person_from_handle(handle)
+        for data in db.iter_raw_person_data():
+            # person = db.get_person_from_handle(handle)
             if user:
                 user.step_progress()
-            if person and self.filt.apply(db, person):
+            if self.filt.apply_to_one(db, data):
                 # store all people in the filter so as to compare later
-                self.with_people.append(person.handle)
+                self.with_people.append(data["handle"])
                 # fill list of ancestor of person if not present yet
-                if handle not in self.ancestor_cache:
+                if data["handle"] not in self.ancestor_cache:
+                    person = from_dict(data)
                     self.add_ancs(db, person)
         if user:
             user.end_progress()
diff --git a/gramps/gen/filters/rules/person/_hasdeath.py b/gramps/gen/filters/rules/person/_hasdeath.py
index d22ba28881..3f6cff14f4 100644
--- a/gramps/gen/filters/rules/person/_hasdeath.py
+++ b/gramps/gen/filters/rules/person/_hasdeath.py
@@ -59,7 +59,8 @@ def prepare(self, db, user):
         else:
             self.date = None
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for event_ref in person.get_event_ref_list():
             if not event_ref:
                 continue
diff --git a/gramps/gen/filters/rules/person/_hasevent.py b/gramps/gen/filters/rules/person/_hasevent.py
index c44bf4d6a0..cad0a5d44f 100644
--- a/gramps/gen/filters/rules/person/_hasevent.py
+++ b/gramps/gen/filters/rules/person/_hasevent.py
@@ -55,10 +55,11 @@ class HasEvent(HasEventBase):
     name = _("People with the personal <event>")
     description = _("Matches people with a personal event of a particular value")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True if a match.
         """
+        person = self.get_object(data)
         for event_ref in person.get_event_ref_list():
             if int(self.list[5]) and event_ref.role != EventRoleType.PRIMARY:
                 # Only match primaries, no witnesses
diff --git a/gramps/gen/filters/rules/person/_hasfamilyattribute.py b/gramps/gen/filters/rules/person/_hasfamilyattribute.py
index e8707b392d..3c9a9d7f71 100644
--- a/gramps/gen/filters/rules/person/_hasfamilyattribute.py
+++ b/gramps/gen/filters/rules/person/_hasfamilyattribute.py
@@ -49,9 +49,10 @@ class HasFamilyAttribute(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
         if not self.list[0]:
             return False
+        person = self.get_object(data)
         for f_id in person.get_family_handle_list():
             f = db.get_family_from_handle(f_id)
             if not f:
diff --git a/gramps/gen/filters/rules/person/_hasfamilyevent.py b/gramps/gen/filters/rules/person/_hasfamilyevent.py
index 6b07bc9ad2..01f85eb538 100644
--- a/gramps/gen/filters/rules/person/_hasfamilyevent.py
+++ b/gramps/gen/filters/rules/person/_hasfamilyevent.py
@@ -67,7 +67,8 @@ def prepare(self, db, user):
         except:
             pass
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for handle in person.get_family_handle_list():
             family = db.get_family_from_handle(handle)
             for event_ref in family.get_event_ref_list():
diff --git a/gramps/gen/filters/rules/person/_hasnameof.py b/gramps/gen/filters/rules/person/_hasnameof.py
index 2ef2c10dc4..484c6f4a75 100644
--- a/gramps/gen/filters/rules/person/_hasnameof.py
+++ b/gramps/gen/filters/rules/person/_hasnameof.py
@@ -63,7 +63,8 @@ class HasNameOf(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for name in [person.get_primary_name()] + person.get_alternate_names():
             if self.match_name(name):
                 return True
diff --git a/gramps/gen/filters/rules/person/_hasnameorigintype.py b/gramps/gen/filters/rules/person/_hasnameorigintype.py
index 9e999495c9..3ef9753879 100644
--- a/gramps/gen/filters/rules/person/_hasnameorigintype.py
+++ b/gramps/gen/filters/rules/person/_hasnameorigintype.py
@@ -61,11 +61,12 @@ def prepare(self, db, user):
             self.name_origin_type = NameOriginType()
             self.name_origin_type.set_from_xml_str(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
         if self.name_origin_type:
+            obj = self.get_object(data)
             for name in [obj.get_primary_name()] + obj.get_alternate_names():
                 for surname in name.get_surname_list():
                     if surname.get_origintype() == self.name_origin_type:
diff --git a/gramps/gen/filters/rules/person/_hasnametype.py b/gramps/gen/filters/rules/person/_hasnametype.py
index 9e8fa7c011..378ee81d4f 100644
--- a/gramps/gen/filters/rules/person/_hasnametype.py
+++ b/gramps/gen/filters/rules/person/_hasnametype.py
@@ -61,11 +61,12 @@ def prepare(self, db, user):
             self.name_type = NameType()
             self.name_type.set_from_xml_str(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
         if self.name_type:
+            obj = self.get_object(data)
             for name in [obj.get_primary_name()] + obj.get_alternate_names():
                 if name.get_type() == self.name_type:
                     return True
diff --git a/gramps/gen/filters/rules/person/_hasnickname.py b/gramps/gen/filters/rules/person/_hasnickname.py
index 48f7da0405..8ab5ee09fc 100644
--- a/gramps/gen/filters/rules/person/_hasnickname.py
+++ b/gramps/gen/filters/rules/person/_hasnickname.py
@@ -48,7 +48,8 @@ class HasNickname(Rule):
     description = _("Matches people with a nickname")
     category = _("General filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         if person.get_nick_name():
             return True
         return False
diff --git a/gramps/gen/filters/rules/person/_hasothergender.py b/gramps/gen/filters/rules/person/_hasothergender.py
index 1d8533f03b..e0f6b0f139 100644
--- a/gramps/gen/filters/rules/person/_hasothergender.py
+++ b/gramps/gen/filters/rules/person/_hasothergender.py
@@ -48,5 +48,5 @@ class HasOtherGender(Rule):
     category = _("General filters")
     description = _("Matches all people with other gender")
 
-    def apply(self, db, person):
-        return person.gender == Person.OTHER
+    def apply_to_one(self, db, data):
+        return data["gender"] == Person.OTHER
diff --git a/gramps/gen/filters/rules/person/_hasrelationship.py b/gramps/gen/filters/rules/person/_hasrelationship.py
index 7223d99100..d9a346cf6f 100644
--- a/gramps/gen/filters/rules/person/_hasrelationship.py
+++ b/gramps/gen/filters/rules/person/_hasrelationship.py
@@ -65,10 +65,11 @@ def prepare(self, db, user):
             self.relationship_type = FamilyRelType()
             self.relationship_type.set_from_xml_str(self.list[1])
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         relationship_type = 0
         total_children = 0
         number_relations = len(obj.get_family_handle_list())
diff --git a/gramps/gen/filters/rules/person/_hassoundexname.py b/gramps/gen/filters/rules/person/_hassoundexname.py
index af3227acd6..7bef3a65cb 100644
--- a/gramps/gen/filters/rules/person/_hassoundexname.py
+++ b/gramps/gen/filters/rules/person/_hassoundexname.py
@@ -65,10 +65,11 @@ def prepare(self, db, user):
         if self.list[0]:
             self.soundex = soundex(self.list[0])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         for name in [obj.get_primary_name()] + obj.get_alternate_names():
             if self._match_name(name):
                 return True
diff --git a/gramps/gen/filters/rules/person/_hastextmatchingsubstringof.py b/gramps/gen/filters/rules/person/_hastextmatchingsubstringof.py
index 7aa4c2aaa7..defc2a984e 100644
--- a/gramps/gen/filters/rules/person/_hastextmatchingsubstringof.py
+++ b/gramps/gen/filters/rules/person/_hastextmatchingsubstringof.py
@@ -80,7 +80,8 @@ def reset(self):
         self.place_map.clear()
         self.media_map.clear()
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         if person.handle in self.person_map:  # Cached by matching Source?
             return True
         if self.match_object(person):  # first match the person itself
diff --git a/gramps/gen/filters/rules/person/_hasunknowngender.py b/gramps/gen/filters/rules/person/_hasunknowngender.py
index 5c25eb8995..e296806ab2 100644
--- a/gramps/gen/filters/rules/person/_hasunknowngender.py
+++ b/gramps/gen/filters/rules/person/_hasunknowngender.py
@@ -48,5 +48,5 @@ class HasUnknownGender(Rule):
     category = _("General filters")
     description = _("Matches all people with unknown gender")
 
-    def apply(self, db, person):
-        return person.gender == Person.UNKNOWN
+    def apply_to_one(self, db, data):
+        return data["gender"] == Person.UNKNOWN
diff --git a/gramps/gen/filters/rules/person/_havealtfamilies.py b/gramps/gen/filters/rules/person/_havealtfamilies.py
index 41c6b46827..1017785b8b 100644
--- a/gramps/gen/filters/rules/person/_havealtfamilies.py
+++ b/gramps/gen/filters/rules/person/_havealtfamilies.py
@@ -46,7 +46,8 @@ class HaveAltFamilies(Rule):
     description = _("Matches people who were adopted")
     category = _("Family filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for fhandle in person.get_parent_family_handle_list():
             family = db.get_family_from_handle(fhandle)
             if family:
diff --git a/gramps/gen/filters/rules/person/_havechildren.py b/gramps/gen/filters/rules/person/_havechildren.py
index 3753d7a8a3..2980c26d26 100644
--- a/gramps/gen/filters/rules/person/_havechildren.py
+++ b/gramps/gen/filters/rules/person/_havechildren.py
@@ -48,7 +48,8 @@ class HaveChildren(Rule):
     description = _("Matches people who have children")
     category = _("Family filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for family_handle in person.get_family_handle_list():
             family = db.get_family_from_handle(family_handle)
             if family is not None and family.get_child_ref_list():
diff --git a/gramps/gen/filters/rules/person/_incompletenames.py b/gramps/gen/filters/rules/person/_incompletenames.py
index 3e11c52b76..28b06ff543 100644
--- a/gramps/gen/filters/rules/person/_incompletenames.py
+++ b/gramps/gen/filters/rules/person/_incompletenames.py
@@ -47,7 +47,8 @@ class IncompleteNames(Rule):
     description = _("Matches people with firstname or lastname missing")
     category = _("General filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for name in [person.get_primary_name()] + person.get_alternate_names():
             if name.get_first_name().strip() == "":
                 return True
diff --git a/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py b/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
index ff7b6ea5bc..74c9fc920f 100644
--- a/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
@@ -24,6 +24,7 @@
 #
 # -------------------------------------------------------------------------
 from ....const import GRAMPS_LOCALE as glocale
+from gramps.gen.lib.serialize import to_dict
 
 _ = glocale.translation.gettext
 
@@ -71,10 +72,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for data in db.iter_raw_person_data():
             if user:
                 user.step_progress()
-            if self.filt.apply(db, person):
+            if self.filt.apply_to_one(db, data):
+                person = from_dict(data)
                 self.init_ancestor_list(db, person, first)
         if user:
             user.end_progress()
@@ -83,5 +85,5 @@ def reset(self):
         self.filt.requestreset()
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/person/_isbookmarked.py b/gramps/gen/filters/rules/person/_isbookmarked.py
index 715cefcca3..7099f49043 100644
--- a/gramps/gen/filters/rules/person/_isbookmarked.py
+++ b/gramps/gen/filters/rules/person/_isbookmarked.py
@@ -51,5 +51,5 @@ class IsBookmarked(Rule):
     def prepare(self, db, user):
         self.bookmarks = db.get_bookmarks().get()
 
-    def apply(self, db, person):
-        return person.handle in self.bookmarks
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.bookmarks
diff --git a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
index fa737f2652..20cd731760 100644
--- a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
@@ -73,8 +73,8 @@ def reset(self):
         self.filt.requestreset()
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person):
         if not person:
diff --git a/gramps/gen/filters/rules/person/_isdefaultperson.py b/gramps/gen/filters/rules/person/_isdefaultperson.py
index 2b58ed43d4..207fe7333a 100644
--- a/gramps/gen/filters/rules/person/_isdefaultperson.py
+++ b/gramps/gen/filters/rules/person/_isdefaultperson.py
@@ -51,9 +51,8 @@ def prepare(self, db, user):
         p = db.get_default_person()
         if p:
             self.def_handle = p.get_handle()
-            self.apply = self.apply_real
         else:
-            self.apply = lambda db, p: False
+            self.def_handle = None
 
-    def apply_real(self, db, person):
-        return person.handle == self.def_handle
+    def apply_to_one(self, db, data):
+        return data["handle"] == self.def_handle
diff --git a/gramps/gen/filters/rules/person/_isdescendantfamilyof.py b/gramps/gen/filters/rules/person/_isdescendantfamilyof.py
index bc288b289e..066e763a08 100644
--- a/gramps/gen/filters/rules/person/_isdescendantfamilyof.py
+++ b/gramps/gen/filters/rules/person/_isdescendantfamilyof.py
@@ -58,7 +58,7 @@ class IsDescendantFamilyOf(Rule):
 
     def prepare(self, db, user):
         self.db = db
-        self.matches = set()
+        self.map = set()
         self.root_person = db.get_person_from_gramps_id(self.list[0])
         self.add_matches(self.root_person)
         try:
@@ -72,10 +72,10 @@ def prepare(self, db, user):
             self.exclude()
 
     def reset(self):
-        self.matches = set()
+        self.map = set()
 
-    def apply(self, db, person):
-        return person.handle in self.matches
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def add_matches(self, person):
         if not person:
@@ -86,10 +86,10 @@ def add_matches(self, person):
 
         while expand:
             person = expand.pop(0)
-            if person is None or person.handle in self.matches:
+            if person is None or person.handle in self.map:
                 # if we have been here before, skip
                 continue
-            self.matches.add(person.handle)
+            self.map.add(person.handle)
             for family_handle in person.get_family_handle_list():
                 family = self.db.get_family_from_handle(family_handle)
                 if family:
@@ -102,13 +102,13 @@ def add_matches(self, person):
                         spouse_handle = family.get_mother_handle()
                     else:
                         spouse_handle = family.get_father_handle()
-                    self.matches.add(spouse_handle)
+                    self.map.add(spouse_handle)
 
     def exclude(self):
         # This removes root person and his/her spouses from the matches set
         if not self.root_person:
             return
-        self.matches.remove(self.root_person.handle)
+        self.map.remove(self.root_person.handle)
         for family_handle in self.root_person.get_family_handle_list():
             family = self.db.get_family_from_handle(family_handle)
             if family:
@@ -116,4 +116,4 @@ def exclude(self):
                     spouse_handle = family.get_mother_handle()
                 else:
                     spouse_handle = family.get_father_handle()
-                self.matches.remove(spouse_handle)
+                self.map.remove(spouse_handle)
diff --git a/gramps/gen/filters/rules/person/_isdescendantof.py b/gramps/gen/filters/rules/person/_isdescendantof.py
index 52cea65c8f..8922f9b0a8 100644
--- a/gramps/gen/filters/rules/person/_isdescendantof.py
+++ b/gramps/gen/filters/rules/person/_isdescendantof.py
@@ -65,8 +65,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person, first):
         if not person or person.handle in self.map:
diff --git a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
index dbb864b4b4..4f3288d996 100644
--- a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
@@ -83,5 +83,5 @@ def reset(self):
         self.filt.requestreset()
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/person/_isduplicatedancestorof.py b/gramps/gen/filters/rules/person/_isduplicatedancestorof.py
index 9d31ca2e20..eaf6ceb98c 100644
--- a/gramps/gen/filters/rules/person/_isduplicatedancestorof.py
+++ b/gramps/gen/filters/rules/person/_isduplicatedancestorof.py
@@ -54,18 +54,18 @@ class IsDuplicatedAncestorOf(Rule):
 
     def prepare(self, db, user):
         self.db = db
+        self.cache = set()
         self.map = set()
-        self.map2 = set()
         root_person = db.get_person_from_gramps_id(self.list[0])
         if root_person:
             self.init_ancestor_list(db, root_person)
 
     def reset(self):
+        self.cache.clear()
         self.map.clear()
-        self.map2.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map2
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_ancestor_list(self, db, person):
         fam_id = person.get_main_parents_family_handle()
@@ -80,10 +80,10 @@ def init_ancestor_list(self, db, person):
                     self.go_deeper(db, db.get_person_from_handle(f_id))
 
     def go_deeper(self, db, person):
-        if person and person.handle in self.map:
-            self.map2.add((person.handle))
+        if person and person.handle in self.cache:
+            self.map.add((person.handle))
             # the following keeps from scanning same parts of tree multiple
             # times and avoids crash on tree loops.
             return
-        self.map.add((person.handle))
+        self.cache.add((person.handle))
         self.init_ancestor_list(db, person)
diff --git a/gramps/gen/filters/rules/person/_isfemale.py b/gramps/gen/filters/rules/person/_isfemale.py
index 2fa21e6e00..ff269ac913 100644
--- a/gramps/gen/filters/rules/person/_isfemale.py
+++ b/gramps/gen/filters/rules/person/_isfemale.py
@@ -48,5 +48,5 @@ class IsFemale(Rule):
     category = _("General filters")
     description = _("Matches all females")
 
-    def apply(self, db, person):
-        return person.gender == Person.FEMALE
+    def apply_to_one(self, db, data):
+        return data["gender"] == Person.FEMALE
diff --git a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorof.py b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorof.py
index c88f8c332f..82b2128dc1 100644
--- a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorof.py
+++ b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorof.py
@@ -87,5 +87,5 @@ def init_ancestor_list(self, root_handle):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofbookmarked.py b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofbookmarked.py
index bbdda54b48..261d249254 100644
--- a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofbookmarked.py
+++ b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofbookmarked.py
@@ -62,11 +62,8 @@ def prepare(self, db, user):
         self.db = db
         bookmarks = db.get_bookmarks().get()
         self.map = set()
-        if len(bookmarks) == 0:
-            self.apply = lambda db, p: False
-        else:
+        if len(bookmarks) != 0:
             self.bookmarks = set(bookmarks)
-            self.apply = self.apply_real
             for self.bookmarkhandle in self.bookmarks:
                 self.init_ancestor_list(self.bookmarkhandle, 1)
 
@@ -95,8 +92,8 @@ def init_ancestor_list(self, handle, gen):
             if m_id:
                 self.init_ancestor_list(m_id, gen + 1)
 
-    def apply_real(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def reset(self):
         self.map.clear()
diff --git a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofdefaultperson.py b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofdefaultperson.py
index 8afef23538..28a18a4d78 100644
--- a/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofdefaultperson.py
+++ b/gramps/gen/filters/rules/person/_islessthannthgenerationancestorofdefaultperson.py
@@ -58,10 +58,9 @@ def prepare(self, db, user):
         p = db.get_default_person()
         if p:
             self.def_handle = p.get_handle()
-            self.apply = self.apply_real
             self.init_ancestor_list(self.def_handle, 1)
         else:
-            self.apply = lambda db, p: False
+            self.def_handle = None
 
     def init_ancestor_list(self, handle, gen):
         #        if p.get_handle() in self.map:
@@ -88,8 +87,8 @@ def init_ancestor_list(self, handle, gen):
             if m_id:
                 self.init_ancestor_list(m_id, gen + 1)
 
-    def apply_real(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def reset(self):
         self.map.clear()
diff --git a/gramps/gen/filters/rules/person/_islessthannthgenerationdescendantof.py b/gramps/gen/filters/rules/person/_islessthannthgenerationdescendantof.py
index d6429edbc0..c6ee2b0ab4 100644
--- a/gramps/gen/filters/rules/person/_islessthannthgenerationdescendantof.py
+++ b/gramps/gen/filters/rules/person/_islessthannthgenerationdescendantof.py
@@ -64,8 +64,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person, gen):
         if not person or person.handle in self.map:
diff --git a/gramps/gen/filters/rules/person/_ismale.py b/gramps/gen/filters/rules/person/_ismale.py
index 1349ea18c9..e0ea087f69 100644
--- a/gramps/gen/filters/rules/person/_ismale.py
+++ b/gramps/gen/filters/rules/person/_ismale.py
@@ -48,5 +48,5 @@ class IsMale(Rule):
     category = _("General filters")
     description = _("Matches all males")
 
-    def apply(self, db, person):
-        return person.gender == Person.MALE
+    def apply_to_one(self, db, data):
+        return data["gender"] == Person.MALE
diff --git a/gramps/gen/filters/rules/person/_ismorethannthgenerationancestorof.py b/gramps/gen/filters/rules/person/_ismorethannthgenerationancestorof.py
index 0391d9bafb..10500fa507 100644
--- a/gramps/gen/filters/rules/person/_ismorethannthgenerationancestorof.py
+++ b/gramps/gen/filters/rules/person/_ismorethannthgenerationancestorof.py
@@ -84,5 +84,5 @@ def init_ancestor_list(self, root_handle):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/person/_ismorethannthgenerationdescendantof.py b/gramps/gen/filters/rules/person/_ismorethannthgenerationdescendantof.py
index 6869df6314..e1494cb04d 100644
--- a/gramps/gen/filters/rules/person/_ismorethannthgenerationdescendantof.py
+++ b/gramps/gen/filters/rules/person/_ismorethannthgenerationdescendantof.py
@@ -64,8 +64,8 @@ def prepare(self, db, user):
     def reset(self):
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person, gen):
         if not person:
diff --git a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
index a262ac23e3..2c7b047df6 100644
--- a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
@@ -73,8 +73,8 @@ def reset(self):
         self.filt.requestreset()
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person):
         for fam_id in person.get_parent_family_handle_list():
diff --git a/gramps/gen/filters/rules/person/_isrelatedwith.py b/gramps/gen/filters/rules/person/_isrelatedwith.py
index c478257ff4..567d0240e0 100644
--- a/gramps/gen/filters/rules/person/_isrelatedwith.py
+++ b/gramps/gen/filters/rules/person/_isrelatedwith.py
@@ -55,17 +55,17 @@ def prepare(self, db, user):
         """
         self.db = db
 
-        self.relatives = []
+        self.map = set()
         self.add_relative(db.get_person_from_gramps_id(self.list[0]))
 
     def reset(self):
-        self.relatives = []
+        self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.relatives
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def add_relative(self, start):
-        """Non-recursive function that scans relatives and add them to self.relatives"""
+        """Non-recursive function that scans relatives and add them to self.map"""
         if not (start):
             return
 
@@ -107,5 +107,5 @@ def add_relative(self, start):
                     for child_ref in family.get_child_ref_list():
                         expand.append(self.db.get_person_from_handle(child_ref.ref))
 
-        self.relatives = list(relatives.keys())
+        self.map = set(list(relatives.keys()))
         return
diff --git a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
index d6974d2d0f..d770cda16b 100644
--- a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
@@ -72,8 +72,8 @@ def reset(self):
         self.matchfilt.requestreset()
         self.map.clear()
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, person):
         if not person:
diff --git a/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py b/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
index c96f88a6ac..d22cf416c9 100644
--- a/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
@@ -54,7 +54,8 @@ def prepare(self, db, user):
         self.filt = MatchesFilter(self.list)
         self.filt.requestprepare(db, user)
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for family_handle in person.get_family_handle_list():
             family = db.get_family_from_handle(family_handle)
             if family:
diff --git a/gramps/gen/filters/rules/person/_iswitness.py b/gramps/gen/filters/rules/person/_iswitness.py
index a7d755c281..7abc8f0d2d 100644
--- a/gramps/gen/filters/rules/person/_iswitness.py
+++ b/gramps/gen/filters/rules/person/_iswitness.py
@@ -62,10 +62,11 @@ def prepare(self, db, user):
             self.event_type = EventType()
             self.event_type.set_from_xml_str(self.list[0])
 
-    def apply(self, db, obj):
+    def apply_to_one(self, db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
         for event_ref in obj.event_ref_list:
             if event_ref.role == EventRoleType.WITNESS:
                 # This is the witness.
diff --git a/gramps/gen/filters/rules/person/_matchidof.py b/gramps/gen/filters/rules/person/_matchidof.py
index 52aca2709c..d3874b78fb 100644
--- a/gramps/gen/filters/rules/person/_matchidof.py
+++ b/gramps/gen/filters/rules/person/_matchidof.py
@@ -48,5 +48,5 @@ class MatchIdOf(Rule):
     description = _("Matches person with a specified Gramps ID")
     category = _("General filters")
 
-    def apply(self, db, person):
-        return person.gramps_id.find(self.list[0]) != -1
+    def apply_to_one(self, db, data):
+        return data["gramps_id"].find(self.list[0]) != -1
diff --git a/gramps/gen/filters/rules/person/_missingparent.py b/gramps/gen/filters/rules/person/_missingparent.py
index 2ee7aea853..41090cfd47 100644
--- a/gramps/gen/filters/rules/person/_missingparent.py
+++ b/gramps/gen/filters/rules/person/_missingparent.py
@@ -50,7 +50,8 @@ class MissingParent(Rule):
     )
     category = _("Family filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         families = person.get_parent_family_handle_list()
         if families == []:
             return True
diff --git a/gramps/gen/filters/rules/person/_multiplemarriages.py b/gramps/gen/filters/rules/person/_multiplemarriages.py
index 30532b2d50..7be0d8fb43 100644
--- a/gramps/gen/filters/rules/person/_multiplemarriages.py
+++ b/gramps/gen/filters/rules/person/_multiplemarriages.py
@@ -45,5 +45,5 @@ class MultipleMarriages(Rule):
     description = _("Matches people who have more than one spouse")
     category = _("Family filters")
 
-    def apply(self, db, person):
-        return len(person.get_family_handle_list()) > 1
+    def apply_to_one(self, db, data):
+        return len(data["family_list"]) > 1
diff --git a/gramps/gen/filters/rules/person/_nevermarried.py b/gramps/gen/filters/rules/person/_nevermarried.py
index 66a9634d2c..88b46f12f9 100644
--- a/gramps/gen/filters/rules/person/_nevermarried.py
+++ b/gramps/gen/filters/rules/person/_nevermarried.py
@@ -45,5 +45,5 @@ class NeverMarried(Rule):
     description = _("Matches people who have no spouse")
     category = _("Family filters")
 
-    def apply(self, db, person):
-        return len(person.get_family_handle_list()) == 0
+    def apply_to_one(self, db, data):
+        return len(data["family_list"]) == 0
diff --git a/gramps/gen/filters/rules/person/_nobirthdate.py b/gramps/gen/filters/rules/person/_nobirthdate.py
index 22e979a15b..be6b999649 100644
--- a/gramps/gen/filters/rules/person/_nobirthdate.py
+++ b/gramps/gen/filters/rules/person/_nobirthdate.py
@@ -45,7 +45,9 @@ class NoBirthdate(Rule):
     description = _("Matches people without a known birthdate")
     category = _("General filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
+        # FIXME: can probably do all of this without object:
         birth_ref = person.get_birth_ref()
         if not birth_ref:
             return True
diff --git a/gramps/gen/filters/rules/person/_nodeathdate.py b/gramps/gen/filters/rules/person/_nodeathdate.py
index 6884d69d9e..56e710d927 100644
--- a/gramps/gen/filters/rules/person/_nodeathdate.py
+++ b/gramps/gen/filters/rules/person/_nodeathdate.py
@@ -45,7 +45,8 @@ class NoDeathdate(Rule):
     description = _("Matches people without a known deathdate")
     category = _("General filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         death_ref = person.get_death_ref()
         if not death_ref:
             return True
diff --git a/gramps/gen/filters/rules/person/_personwithincompleteevent.py b/gramps/gen/filters/rules/person/_personwithincompleteevent.py
index 58ce20d7b3..d0ddcb9df2 100644
--- a/gramps/gen/filters/rules/person/_personwithincompleteevent.py
+++ b/gramps/gen/filters/rules/person/_personwithincompleteevent.py
@@ -45,7 +45,8 @@ class PersonWithIncompleteEvent(Rule):
     description = _("Matches people with missing date or place in an event")
     category = _("Event filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for event_ref in person.get_event_ref_list():
             if event_ref:
                 event = db.get_event_from_handle(event_ref.ref)
diff --git a/gramps/gen/filters/rules/person/_probablyalive.py b/gramps/gen/filters/rules/person/_probablyalive.py
index 91d9e80b69..2705141c43 100644
--- a/gramps/gen/filters/rules/person/_probablyalive.py
+++ b/gramps/gen/filters/rules/person/_probablyalive.py
@@ -54,5 +54,6 @@ def prepare(self, db, user):
         except:
             self.current_date = None
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(person)
         return probably_alive(person, db, self.current_date)
diff --git a/gramps/gen/filters/rules/person/_regexpname.py b/gramps/gen/filters/rules/person/_regexpname.py
index 95aa6dc274..48d3da049e 100644
--- a/gramps/gen/filters/rules/person/_regexpname.py
+++ b/gramps/gen/filters/rules/person/_regexpname.py
@@ -53,7 +53,8 @@ class RegExpName(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
+        person = self.get_object(data)
         for name in [person.get_primary_name()] + person.get_alternate_names():
             for field in [
                 name.first_name,
diff --git a/gramps/gen/filters/rules/person/_relationshippathbetween.py b/gramps/gen/filters/rules/person/_relationshippathbetween.py
index 39b786afd4..881d5614c3 100644
--- a/gramps/gen/filters/rules/person/_relationshippathbetween.py
+++ b/gramps/gen/filters/rules/person/_relationshippathbetween.py
@@ -95,8 +95,8 @@ def apply_filter(self, rank, handle, plist, pmap):
             self.apply_filter(rank + 1, family.get_father_handle(), plist, pmap)
             self.apply_filter(rank + 1, family.get_mother_handle(), plist, pmap)
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
 
     def init_list(self, p1_handle, p2_handle):
         firstMap = {}
diff --git a/gramps/gen/filters/rules/person/_relationshippathbetweenbookmarks.py b/gramps/gen/filters/rules/person/_relationshippathbetweenbookmarks.py
index 9c4619523a..1cac75e2ee 100644
--- a/gramps/gen/filters/rules/person/_relationshippathbetweenbookmarks.py
+++ b/gramps/gen/filters/rules/person/_relationshippathbetweenbookmarks.py
@@ -166,5 +166,5 @@ def init_list(self):
                 except:
                     pass
 
-    def apply(self, db, person):
-        return person.handle in self.map
+    def apply_to_one(self, db, data):
+        return data["handle"] in self.map
diff --git a/gramps/gen/filters/rules/person/_searchname.py b/gramps/gen/filters/rules/person/_searchname.py
index f6b62be8d5..1173d93e38 100644
--- a/gramps/gen/filters/rules/person/_searchname.py
+++ b/gramps/gen/filters/rules/person/_searchname.py
@@ -49,11 +49,12 @@ class SearchName(Rule):
     description = _("Matches people with a specified (partial) name")
     category = _("General filters")
 
-    def apply(self, db, person):
+    def apply_to_one(self, db, data):
         src = self.list[0].upper()
         if not src:
             return False
 
+        person = self.get_object(data)
         for name in [person.get_primary_name()] + person.get_alternate_names():
             for field in [
                 name.first_name,
diff --git a/gramps/gen/filters/rules/place/_hasdata.py b/gramps/gen/filters/rules/place/_hasdata.py
index cf431af5ec..1ceab2d410 100644
--- a/gramps/gen/filters/rules/place/_hasdata.py
+++ b/gramps/gen/filters/rules/place/_hasdata.py
@@ -67,7 +67,8 @@ def prepare(self, db, user):
             self.place_type = PlaceType()
             self.place_type.set_from_xml_str(self.list[1])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
+        obj = self.get_object(data)
         """
         Apply the rule. Return True on a match.
         """
diff --git a/gramps/gen/filters/rules/place/_hasnolatorlon.py b/gramps/gen/filters/rules/place/_hasnolatorlon.py
index 768592d3c5..c3f7153d3e 100644
--- a/gramps/gen/filters/rules/place/_hasnolatorlon.py
+++ b/gramps/gen/filters/rules/place/_hasnolatorlon.py
@@ -50,7 +50,7 @@ class HasNoLatOrLon(Rule):
     description = _("Matches places with empty latitude or longitude")
     category = _("Position filters")
 
-    def apply(self, db, place):
-        if place.get_latitude().strip and place.get_longitude().strip():
+    def apply_to_one(self, db, data):
+        if data["lat"].strip and data["long"].strip():
             return False
         return True
diff --git a/gramps/gen/filters/rules/place/_hasplace.py b/gramps/gen/filters/rules/place/_hasplace.py
index 544a1cc8f2..4e854a7b0d 100644
--- a/gramps/gen/filters/rules/place/_hasplace.py
+++ b/gramps/gen/filters/rules/place/_hasplace.py
@@ -73,7 +73,8 @@ class HasPlace(Rule):
         PlaceType.PARISH: 8,
     }
 
-    def apply(self, db, place):
+    def apply_to_one(self, db, data):
+        place = self.get_object(data)
         if not self.match_substring(0, place.get_title()):
             return False
 
diff --git a/gramps/gen/filters/rules/place/_hastitle.py b/gramps/gen/filters/rules/place/_hastitle.py
index 6820e480e8..b60f6d7e5f 100644
--- a/gramps/gen/filters/rules/place/_hastitle.py
+++ b/gramps/gen/filters/rules/place/_hastitle.py
@@ -53,7 +53,8 @@ class HasTitle(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, place):
+    def apply_to_one(self, db, data):
+        place = self.get_object(data)
         if not self.match_substring(0, displayer.display(db, place)):
             return False
         return True
diff --git a/gramps/gen/filters/rules/place/_inlatlonneighborhood.py b/gramps/gen/filters/rules/place/_inlatlonneighborhood.py
index c3c6bfa0b4..e55b7ac631 100644
--- a/gramps/gen/filters/rules/place/_inlatlonneighborhood.py
+++ b/gramps/gen/filters/rules/place/_inlatlonneighborhood.py
@@ -131,7 +131,7 @@ def prepare(self, db, user):
                     self.E2 = 180.0
                     self.W = -180
 
-    def apply(self, db, place):
+    def apply_to_one(self, db, data):
         if self.halfheight == -1 and self.halfwidth == -1:
             return False
 
@@ -145,6 +145,8 @@ def apply(self, db, place):
 
         # now we know at least one is given in the filter and is valid
 
+        place = self.get_object(data)
+        # FIXME: rest can probably be done in data:
         # the place we look at must have lat AND lon entered
         if not (place.get_latitude().strip and place.get_longitude().strip()):
             return False
diff --git a/gramps/gen/filters/rules/place/_isenclosedby.py b/gramps/gen/filters/rules/place/_isenclosedby.py
index 7be1b88465..a59c3a2932 100644
--- a/gramps/gen/filters/rules/place/_isenclosedby.py
+++ b/gramps/gen/filters/rules/place/_isenclosedby.py
@@ -58,11 +58,11 @@ def prepare(self, db, user):
         if place:
             self.handle = place.handle
 
-    def apply(self, db, place):
+    def apply_to_one(self, db, data):
         if self.handle is None:
             return False
-        if self.list[1] == "1" and place.handle == self.handle:
+        if self.list[1] == "1" and data["handle"] == self.handle:
             return True
-        if located_in(db, place.handle, self.handle):
+        if located_in(db, data["handle"], self.handle):
             return True
         return False
diff --git a/gramps/gen/filters/rules/place/_matcheseventfilter.py b/gramps/gen/filters/rules/place/_matcheseventfilter.py
index b5a8745d25..510124ef23 100644
--- a/gramps/gen/filters/rules/place/_matcheseventfilter.py
+++ b/gramps/gen/filters/rules/place/_matcheseventfilter.py
@@ -58,12 +58,13 @@ class MatchesEventFilter(MatchesFilterBase):
     # we want to have this filter show event filters
     namespace = "Event"
 
-    def apply(self, db, event):
+    def apply_to_one(self, db, data):
         filt = self.find_filter()
         if filt:
             for classname, handle in db.find_backlink_handles(
-                event.get_handle(), ["Event"]
+                data["handle"], ["Event"]
             ):
-                if filt.check(db, handle):
+                event_data = db.get_raw_event_data(handle)
+                if filt.apply_to_one(db, event_data):
                     return True
         return False
diff --git a/gramps/gen/filters/rules/place/_withinarea.py b/gramps/gen/filters/rules/place/_withinarea.py
index 6f31228535..759c0298fc 100644
--- a/gramps/gen/filters/rules/place/_withinarea.py
+++ b/gramps/gen/filters/rules/place/_withinarea.py
@@ -106,16 +106,16 @@ def prepare(self, db, user):
                 self.radius = float(value)
             self.radius = self.radius / 2
 
-    def apply(self, dummy_db, place):
+    def apply_to_one(self, dummy_db, data):
         if self.handle is None:
             return False
         if self.latitude is None:
             return False
         if self.longitude is None:
             return False
-        if place:
-            lat = place.get_latitude()
-            lon = place.get_longitude()
+        if data:
+            lat = data["lat"]
+            lon = data["long"]
         if lat and lon:
             latit, longit = conv_lat_lon(lat, lon, "D.D8")
             if latit is None or longit is None:
diff --git a/gramps/gen/filters/rules/repository/_hasrepo.py b/gramps/gen/filters/rules/repository/_hasrepo.py
index e2afb39add..140e54ac30 100644
--- a/gramps/gen/filters/rules/repository/_hasrepo.py
+++ b/gramps/gen/filters/rules/repository/_hasrepo.py
@@ -67,10 +67,12 @@ def prepare(self, db, user):
             self.rtype = RepositoryType()
             self.rtype.set_from_xml_str(self.list[1])
 
-    def apply(self, _db, obj):
+    def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True on a match.
         """
+        obj = self.get_object(data)
+        # FIXME: probably can do the rest without object:
         if not self.match_substring(0, obj.get_name()):
             return False
 
diff --git a/gramps/gen/filters/rules/repository/_matchesnamesubstringof.py b/gramps/gen/filters/rules/repository/_matchesnamesubstringof.py
index 8ae45db63c..876e90fdf1 100644
--- a/gramps/gen/filters/rules/repository/_matchesnamesubstringof.py
+++ b/gramps/gen/filters/rules/repository/_matchesnamesubstringof.py
@@ -47,6 +47,6 @@ class MatchesNameSubstringOf(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, repository):
+    def apply_to_one(self, db, data):
         """Apply the filter"""
-        return self.match_substring(0, repository.get_name())
+        return self.match_substring(0, data["name"])
diff --git a/gramps/gen/filters/rules/source/_hasrepository.py b/gramps/gen/filters/rules/source/_hasrepository.py
index e1c9c8e626..a23fc4d0f4 100644
--- a/gramps/gen/filters/rules/source/_hasrepository.py
+++ b/gramps/gen/filters/rules/source/_hasrepository.py
@@ -61,8 +61,8 @@ def prepare(self, db, user):
 
         self.userSelectedCount = int(self.list[0])
 
-    def apply(self, db, obj):
-        count = len(obj.get_reporef_list())
+    def apply_to_one(self, db, data):
+        count = len(data["reporef_list"])
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
         elif self.count_type == 2:  # "greater than"
diff --git a/gramps/gen/filters/rules/source/_hasrepositorycallnumberref.py b/gramps/gen/filters/rules/source/_hasrepositorycallnumberref.py
index 090172cf65..647274dd86 100644
--- a/gramps/gen/filters/rules/source/_hasrepositorycallnumberref.py
+++ b/gramps/gen/filters/rules/source/_hasrepositorycallnumberref.py
@@ -51,8 +51,8 @@ class HasRepositoryCallNumberRef(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, obj):
-        for repo_ref in obj.get_reporef_list():
-            if self.match_substring(0, repo_ref.call_number):
+    def apply_to_one(self, db, data):
+        for repo_ref in data["reporef_list"]:
+            if self.match_substring(0, repo_ref["call_number"]):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/source/_matchesrepositoryfilter.py b/gramps/gen/filters/rules/source/_matchesrepositoryfilter.py
index 2c0e8f1822..7b9a60ebb8 100644
--- a/gramps/gen/filters/rules/source/_matchesrepositoryfilter.py
+++ b/gramps/gen/filters/rules/source/_matchesrepositoryfilter.py
@@ -57,13 +57,14 @@ def prepare(self, db, user):
         MatchesFilterBase.prepare(self, db, user)
         self.MRF_filt = self.find_filter()
 
-    def apply(self, db, object):
+    def apply_to_one(self, db, data):
         if self.MRF_filt is None:
             return False
 
-        repolist = [x.ref for x in object.get_reporef_list()]
+        repolist = [x.ref for x in data["reporef_list"]]
         for repohandle in repolist:
             # check if repo in repository filter
-            if self.MRF_filt.check(db, repohandle):
+            repo_data = self.get_raw_repository_data(repohandle)
+            if self.MRF_filt.apply_to_one(db, repo_data):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/source/_matchestitlesubstringof.py b/gramps/gen/filters/rules/source/_matchestitlesubstringof.py
index e88e70c736..51ebadfd50 100644
--- a/gramps/gen/filters/rules/source/_matchestitlesubstringof.py
+++ b/gramps/gen/filters/rules/source/_matchestitlesubstringof.py
@@ -47,6 +47,6 @@ class MatchesTitleSubstringOf(Rule):
     category = _("General filters")
     allow_regex = True
 
-    def apply(self, db, source):
+    def apply_to_one(self, db, data):
         """Apply the filter"""
-        return self.match_substring(0, source.get_title())
+        return self.match_substring(0, data["title"])

From 212fe31577bf1ee5956694a6d4aa308d5d002a6d Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 12:01:41 -0500
Subject: [PATCH 22/46] Typos

---
 gramps/gen/filters/_genericfilter.py              | 6 ++----
 gramps/gen/filters/rules/person/_probablyalive.py | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index bf5164d80e..7c72575b5a 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -151,7 +151,7 @@ def walk_filters(self, filter, result):
         if rules:
             result.append((filter.invert, filter.logical_op, rules))
 
-    def apply_to_one_logical_op_to_all(
+    def apply_logical_op_to_all(
         self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
     ):
         final_list = []
@@ -260,9 +260,7 @@ def apply_to_one(self, db, data):
             raise Exception("invalid operator: %r" % self.logical_op)
         return res != self.invert
 
-    def apply_to_one_to_all(
-        self, db, id_list=None, tupleind=None, user=None, tree=False
-    ):
+    def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
         """
         Apply the filter using db.
         If id_list given, the handles in id_list are used. If not given
diff --git a/gramps/gen/filters/rules/person/_probablyalive.py b/gramps/gen/filters/rules/person/_probablyalive.py
index 2705141c43..cbbedd0efb 100644
--- a/gramps/gen/filters/rules/person/_probablyalive.py
+++ b/gramps/gen/filters/rules/person/_probablyalive.py
@@ -55,5 +55,5 @@ def prepare(self, db, user):
             self.current_date = None
 
     def apply_to_one(self, db, data):
-        person = self.get_object(person)
+        person = self.get_object(data)
         return probably_alive(person, db, self.current_date)

From 139de6a72eaf1cd9a2e339f6a85846b26a25d8d4 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 14:39:47 -0500
Subject: [PATCH 23/46] Fixes from big conversion

---
 gramps/gen/filters/_genericfilter.py          | 59 ++++++++++++++++---
 gramps/gen/filters/rules/family/_hasevent.py  |  6 +-
 .../gen/filters/rules/family/_memberbase.py   | 28 ++++-----
 gramps/plugins/lib/librecords.py              |  1 +
 4 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 7c72575b5a..8905b0f55d 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -24,6 +24,8 @@
 Package providing filtering framework for Gramps.
 """
 
+import logging
+
 # ------------------------------------------------------------------------
 #
 # Gramps imports
@@ -44,6 +46,7 @@
 from .rules import Rule
 
 _ = glocale.translation.gettext
+LOG = logging.getLogger(".filter.optimize")
 
 
 # -------------------------------------------------------------------------
@@ -137,19 +140,31 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_people()
 
-    def walk_filters(self, filter, result):
+    def walk_filters(self, filter, parent_invert, result):
         """
         Walk all of the filters/rules and get
         rules with maps
         """
+        LOG.debug(
+            "walking, filter: %s, invert=%s, parent_invert=%s",
+            filter, filter.invert, parent_invert
+        )
         rules = []
         for item in filter.flist:
             if hasattr(item, "find_filter"):
-                self.walk_filters(item.find_filter(), result)
+                self.walk_filters(
+                    item.find_filter(),
+                    not parent_invert if filter.invert else parent_invert,
+                    result
+                )
             elif hasattr(item, "map"):
                 rules.append(item.map)
         if rules:
-            result.append((filter.invert, filter.logical_op, rules))
+            LOG.debug(
+                "filter %s: parent_invert=%s, invert=%s, op=%s, number of maps=%s",
+                filter, parent_invert, filter.invert, filter.logical_op, len(rules)
+            )
+            result.append((parent_invert, filter.logical_op, rules))
 
     def apply_logical_op_to_all(
         self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
@@ -171,22 +186,24 @@ def intersection(sets):
         # Optimiziation
         # ------------------------------------------------------
         all_maps = []
-        self.walk_filters(self, all_maps)
+        self.walk_filters(self, False, all_maps)
+        LOG.debug("number of maps to consider: %s", len(all_maps))
         handles = None
         # Get all positive non-inverted maps
         for inverted, logical_op, maps in all_maps:
             if logical_op == "and" and not inverted:
+                LOG.debug("optimizer match!")
                 if handles is None:
                     handles = intersection(maps)
                 else:
                     handles = intersection([handles] + maps)
         # ------------------------------------------------------
-
+        LOG.debug("optimizer handles: %s", len(handles) if handles else 0)
         if id_list is None:
             if handles:
                 # Use these rather than going through entire database
                 for handle in handles:
-                    json_data = db.get_raw_person_data(handle)
+                    json_data = self.get_raw_data(db, handle)
 
                     if user:
                         user.step_progress()
@@ -215,7 +232,7 @@ def intersection(sets):
                 if handles and handle not in handles:
                     continue
 
-                json_data = db.get_raw_person_data(handle)
+                json_data = self.get_raw_data(db, handle)
 
                 if user:
                     user.step_progress()
@@ -299,6 +316,9 @@ def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
 
         return res
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_person_data(handle)
+
 
 class GenericFamilyFilter(GenericFilter):
     def __init__(self, source=None):
@@ -316,6 +336,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_families()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_family_data(handle)
+
 
 class GenericEventFilter(GenericFilter):
     def __init__(self, source=None):
@@ -333,6 +356,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_events()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_event_data(handle)
+
 
 class GenericSourceFilter(GenericFilter):
     def __init__(self, source=None):
@@ -350,6 +376,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_sources()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_source_data(handle)
+
 
 class GenericCitationFilter(GenericFilter):
     def __init__(self, source=None):
@@ -370,6 +399,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_citations()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_citation_data(handle)
+
 
 class GenericPlaceFilter(GenericFilter):
     def __init__(self, source=None):
@@ -390,6 +422,10 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_places()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_place_data(handle)
+
+
 
 class GenericMediaFilter(GenericFilter):
     def __init__(self, source=None):
@@ -407,6 +443,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_media()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_media_data(handle)
+
 
 class GenericRepoFilter(GenericFilter):
     def __init__(self, source=None):
@@ -424,6 +463,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_repositories()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_repository_data(handle)
+
 
 class GenericNoteFilter(GenericFilter):
     def __init__(self, source=None):
@@ -441,6 +483,9 @@ def find_from_handle(self, db, handle):
     def get_number(self, db):
         return db.get_number_of_notes()
 
+    def get_raw_data(self, db, handle):
+        return db.get_raw_note_data(handle)
+
 
 def GenericFilterFactory(namespace):
     if namespace == "Person":
diff --git a/gramps/gen/filters/rules/family/_hasevent.py b/gramps/gen/filters/rules/family/_hasevent.py
index 43abf83d00..28ed71bf0c 100644
--- a/gramps/gen/filters/rules/family/_hasevent.py
+++ b/gramps/gen/filters/rules/family/_hasevent.py
@@ -58,10 +58,10 @@ class HasEvent(HasEventBase):
 
     def apply_to_one(self, dbase, data):
         family = self.get_object(data)
-        for event_ref in family.get_event_ref_list():
+        for event_ref in data["event_ref_list"]
             if not event_ref:
                 continue
-            event = dbase.get_event_from_handle(event_ref.ref)
-            if HasEventBase.apply(self, dbase, event):
+            event_data = dbase.get_raw_event_data(event_ref["ref"])
+            if HasEventBase.apply_to_one(self, dbase, event_data):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/family/_memberbase.py b/gramps/gen/filters/rules/family/_memberbase.py
index 086d3047bd..32b0098b5d 100644
--- a/gramps/gen/filters/rules/family/_memberbase.py
+++ b/gramps/gen/filters/rules/family/_memberbase.py
@@ -33,31 +33,29 @@
 
 
 def father_base(self, db, data):
-    family = self.get_object(data)
-    father_handle = family.get_father_handle()
+    father_handle = data["father_handle"]
     if father_handle:
-        father = db.get_person_from_handle(father_handle)
-        if father:
-            return self.base_class.apply(self, db, father)
+        father_data = db.get_raw_person_data(father_handle)
+        if father_data:
+            return self.base_class.apply_to_one(self, db, father_data)
         else:
             return False
 
 
 def mother_base(self, db, data):
-    family = self.get_object(data)
-    mother_handle = family.get_mother_handle()
+    mother_handle = data["mother_handle"]
     if mother_handle:
-        mother = db.get_person_from_handle(mother_handle)
-        if mother:
-            return self.base_class.apply(self, db, mother)
+        mother_data = db.get_raw_person_data(mother_handle)
+        if mother_data:
+            return self.base_class.apply_to_one(self, db, mother_data)
         else:
             return False
 
 
 def child_base(self, db, data):
-    family = self.get_object(data)
-    for child_ref in family.get_child_ref_list():
-        child = db.get_person_from_handle(child_ref.ref)
-        if self.base_class.apply(self, db, child):
-            return True
+    for child_ref in data["child_ref_list"]:
+        child_data = db.get_raw_person_data(child_ref["ref"])
+        if child_data:
+            if self.base_class.apply_to_one(self, db, child_data):
+                return True
     return False
diff --git a/gramps/plugins/lib/librecords.py b/gramps/plugins/lib/librecords.py
index 8b706c3985..cc192ccda4 100644
--- a/gramps/plugins/lib/librecords.py
+++ b/gramps/plugins/lib/librecords.py
@@ -386,6 +386,7 @@ def get_unfiltered_person_from_handle(person_handle):
         # Test if either father or mother are in filter
         if filter:
             # we don't want many progress reports popping up, so no user=user
+            # FIXME:
             if not filter.apply_to_all(db, [father_handle, mother_handle]):
                 continue
 

From ad0472c1c746bcfcd52a7f9558ef85d1cf957998 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 14:40:30 -0500
Subject: [PATCH 24/46] Fixes from big conversion; linting

---
 gramps/gen/filters/_genericfilter.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 8905b0f55d..567fd05de8 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -147,7 +147,9 @@ def walk_filters(self, filter, parent_invert, result):
         """
         LOG.debug(
             "walking, filter: %s, invert=%s, parent_invert=%s",
-            filter, filter.invert, parent_invert
+            filter,
+            filter.invert,
+            parent_invert,
         )
         rules = []
         for item in filter.flist:
@@ -155,14 +157,18 @@ def walk_filters(self, filter, parent_invert, result):
                 self.walk_filters(
                     item.find_filter(),
                     not parent_invert if filter.invert else parent_invert,
-                    result
+                    result,
                 )
             elif hasattr(item, "map"):
                 rules.append(item.map)
         if rules:
             LOG.debug(
                 "filter %s: parent_invert=%s, invert=%s, op=%s, number of maps=%s",
-                filter, parent_invert, filter.invert, filter.logical_op, len(rules)
+                filter,
+                parent_invert,
+                filter.invert,
+                filter.logical_op,
+                len(rules),
             )
             result.append((parent_invert, filter.logical_op, rules))
 
@@ -426,7 +432,6 @@ def get_raw_data(self, db, handle):
         return db.get_raw_place_data(handle)
 
 
-
 class GenericMediaFilter(GenericFilter):
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)

From 5481af045afd8ed149669720424614bf41742bc4 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 15:12:48 -0500
Subject: [PATCH 25/46] Fixes and tweaks

---
 gramps/gen/filters/_genericfilter.py           | 10 ++++++++--
 gramps/gen/filters/rules/family/_hasevent.py   |  2 +-
 gramps/gen/filters/rules/media/_hastag.py      |  1 +
 gramps/gen/filters/rules/note/_hastag.py       |  1 +
 gramps/gen/filters/rules/person/_hasevent.py   | 12 +++++++-----
 gramps/gen/filters/rules/place/_hastag.py      |  1 +
 gramps/gen/filters/rules/repository/_hastag.py |  1 +
 gramps/gen/filters/rules/source/_hastag.py     |  1 +
 gramps/gui/views/treemodels/placemodel.py      |  2 +-
 9 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 567fd05de8..45763b7961 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -170,7 +170,13 @@ def walk_filters(self, filter, parent_invert, result):
                 filter.logical_op,
                 len(rules),
             )
-            result.append((parent_invert, filter.logical_op, rules))
+            result.append(
+                (
+                    parent_invert if not filter.invert else not parent_invert,
+                    filter.logical_op,
+                    rules,
+                )
+            )
 
     def apply_logical_op_to_all(
         self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
@@ -192,7 +198,7 @@ def intersection(sets):
         # Optimiziation
         # ------------------------------------------------------
         all_maps = []
-        self.walk_filters(self, False, all_maps)
+        self.walk_filters(self, self.invert, all_maps)
         LOG.debug("number of maps to consider: %s", len(all_maps))
         handles = None
         # Get all positive non-inverted maps
diff --git a/gramps/gen/filters/rules/family/_hasevent.py b/gramps/gen/filters/rules/family/_hasevent.py
index 28ed71bf0c..1292650846 100644
--- a/gramps/gen/filters/rules/family/_hasevent.py
+++ b/gramps/gen/filters/rules/family/_hasevent.py
@@ -58,7 +58,7 @@ class HasEvent(HasEventBase):
 
     def apply_to_one(self, dbase, data):
         family = self.get_object(data)
-        for event_ref in data["event_ref_list"]
+        for event_ref in data["event_ref_list"]:
             if not event_ref:
                 continue
             event_data = dbase.get_raw_event_data(event_ref["ref"])
diff --git a/gramps/gen/filters/rules/media/_hastag.py b/gramps/gen/filters/rules/media/_hastag.py
index 4ee499396c..3a009438d8 100644
--- a/gramps/gen/filters/rules/media/_hastag.py
+++ b/gramps/gen/filters/rules/media/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Media objects with the <tag>")
     description = _("Matches media objects with the particular tag")
+    table = "media"
diff --git a/gramps/gen/filters/rules/note/_hastag.py b/gramps/gen/filters/rules/note/_hastag.py
index ec3044dd52..e76f575f04 100644
--- a/gramps/gen/filters/rules/note/_hastag.py
+++ b/gramps/gen/filters/rules/note/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Notes with the <tag>")
     description = _("Matches notes with the particular tag")
+    table = "note"
diff --git a/gramps/gen/filters/rules/person/_hasevent.py b/gramps/gen/filters/rules/person/_hasevent.py
index cad0a5d44f..7e4c06234d 100644
--- a/gramps/gen/filters/rules/person/_hasevent.py
+++ b/gramps/gen/filters/rules/person/_hasevent.py
@@ -59,12 +59,14 @@ def apply_to_one(self, db, data):
         """
         Apply the rule. Return True if a match.
         """
-        person = self.get_object(data)
-        for event_ref in person.get_event_ref_list():
-            if int(self.list[5]) and event_ref.role != EventRoleType.PRIMARY:
+        for event_ref in data["event_ref_list"]:
+            if (
+                int(self.list[5])
+                and event_ref["role"]["value"] != EventRoleType.PRIMARY
+            ):
                 # Only match primaries, no witnesses
                 continue
-            event = db.get_event_from_handle(event_ref.ref)
-            if HasEventBase.apply(self, db, event):
+            event_data = db.get_raw_event_data(event_ref["ref"])
+            if HasEventBase.apply_to_one(self, db, event_data):
                 return True
         return False
diff --git a/gramps/gen/filters/rules/place/_hastag.py b/gramps/gen/filters/rules/place/_hastag.py
index 21cfb90aa7..0404afe9ae 100644
--- a/gramps/gen/filters/rules/place/_hastag.py
+++ b/gramps/gen/filters/rules/place/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Places with the <tag>")
     description = _("Matches places with the particular tag")
+    table = "place"
diff --git a/gramps/gen/filters/rules/repository/_hastag.py b/gramps/gen/filters/rules/repository/_hastag.py
index e1c81a3a17..0e0ce2de27 100644
--- a/gramps/gen/filters/rules/repository/_hastag.py
+++ b/gramps/gen/filters/rules/repository/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Repositories with the <tag>")
     description = _("Matches repositories with the particular tag")
+    table = "repository"
diff --git a/gramps/gen/filters/rules/source/_hastag.py b/gramps/gen/filters/rules/source/_hastag.py
index 70047d906b..e1d9aa206c 100644
--- a/gramps/gen/filters/rules/source/_hastag.py
+++ b/gramps/gen/filters/rules/source/_hastag.py
@@ -51,3 +51,4 @@ class HasTag(HasTagBase):
     labels = [_("Tag:")]
     name = _("Sources with the <tag>")
     description = _("Matches sources with the particular tag")
+    table = "source"
diff --git a/gramps/gui/views/treemodels/placemodel.py b/gramps/gui/views/treemodels/placemodel.py
index 07c1129f6a..2ee0d9f25e 100644
--- a/gramps/gui/views/treemodels/placemodel.py
+++ b/gramps/gui/views/treemodels/placemodel.py
@@ -139,7 +139,7 @@ def column_name(self, data):
     def search_name(self, data):
         """The search name includes all alt names to enable finding by alt name"""
         return ",".join(
-            [data["name"]["value"]] + [name["value"] for name in data["alt_name"]]
+            [data["name"]["value"]] + [name["value"] for name in data["alt_names"]]
         )
 
     def column_longitude(self, data):

From 401ed14a2142d83143228f3d036f88e4a6459682 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 15:24:39 -0500
Subject: [PATCH 26/46] Fix

---
 gramps/gen/filters/rules/citation/_hassource.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gramps/gen/filters/rules/citation/_hassource.py b/gramps/gen/filters/rules/citation/_hassource.py
index 32d545092c..f908a1b21b 100644
--- a/gramps/gen/filters/rules/citation/_hassource.py
+++ b/gramps/gen/filters/rules/citation/_hassource.py
@@ -54,7 +54,7 @@ class HasSource(HasSourceBase):
 
     def apply_to_one(self, dbase, data):
         citation = self.get_object(data)
-        source = dbase.get_source_from_handle(citation.get_reference_handle())
-        if HasSourceBase.apply_to_one(self, dbase, source.handle):
+        source_data = dbase.get_raw_source_data(citation.get_reference_handle())
+        if HasSourceBase.apply_to_one(self, dbase, source_data):
             return True
         return False

From f6d5841f5007eba09a11948be8eaa423bd865aed Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 16:10:26 -0500
Subject: [PATCH 27/46] Remove all inverted and-maps from handles

---
 gramps/gen/filters/_genericfilter.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 45763b7961..ae230df903 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -194,6 +194,15 @@ def intersection(sets):
             else:
                 return set()
 
+        def union(sets):
+            if sets:
+                result = sets[0]
+                for s in sets[1:]:
+                    result = result.union(s)
+                return result
+            else:
+                return set()
+
         # ------------------------------------------------------
         # Optimiziation
         # ------------------------------------------------------
@@ -204,15 +213,22 @@ def intersection(sets):
         # Get all positive non-inverted maps
         for inverted, logical_op, maps in all_maps:
             if logical_op == "and" and not inverted:
-                LOG.debug("optimizer match!")
+                LOG.debug("optimizer positive match!")
                 if handles is None:
                     handles = intersection(maps)
                 else:
                     handles = intersection([handles] + maps)
+        if handles is not None:
+            # Get all inverted maps and remove from handles
+            for inverted, logical_op, maps in all_maps:
+                if logical_op == "and" and inverted:
+                    LOG.debug("optimizer inverted match!")
+                    not_handles = union(maps)
+                    handles = handles - not_handles
         # ------------------------------------------------------
         LOG.debug("optimizer handles: %s", len(handles) if handles else 0)
         if id_list is None:
-            if handles:
+            if handles is not None:
                 # Use these rather than going through entire database
                 for handle in handles:
                     json_data = self.get_raw_data(db, handle)
@@ -241,7 +257,7 @@ def intersection(sets):
                 else:
                     handle = data[tupleind]
 
-                if handles and handle not in handles:
+                if handles is not None and handle not in handles:
                     continue
 
                 json_data = self.get_raw_data(db, handle)

From 8feb8c37154232ef54743c17df4d48a9775a77f5 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sat, 16 Nov 2024 16:31:24 -0500
Subject: [PATCH 28/46] Another round of fixes

---
 .../filters/rules/person/_deeprelationshippathbetween.py   | 7 +++----
 gramps/gen/filters/rules/person/_ischildoffiltermatch.py   | 7 ++++---
 .../rules/person/_isdescendantfamilyoffiltermatch.py       | 7 ++++---
 .../gen/filters/rules/person/_isdescendantoffiltermatch.py | 7 ++++---
 gramps/gen/filters/rules/person/_isparentoffiltermatch.py  | 7 ++++---
 gramps/gen/filters/rules/person/_issiblingoffiltermatch.py | 7 ++++---
 gramps/gen/filters/rules/person/_isspouseoffiltermatch.py  | 3 ++-
 gramps/gen/filters/rules/test/event_rules_test.py          | 2 +-
 gramps/gen/filters/rules/test/family_rules_test.py         | 2 +-
 gramps/gen/filters/rules/test/media_rules_test.py          | 2 +-
 gramps/gen/filters/rules/test/note_rules_test.py           | 2 +-
 gramps/gen/filters/rules/test/person_rules_test.py         | 2 +-
 gramps/gen/filters/rules/test/place_rules_test.py          | 2 +-
 gramps/gen/filters/rules/test/repository_rules_test.py     | 2 +-
 14 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
index eed5415be0..938d9cb0c8 100644
--- a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
+++ b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
@@ -157,10 +157,9 @@ def prepare(self, db, user):
                 db.get_number_of_people(),
             )
         target_people = []
-        for handle in db.iter_person_handles():
-            person = db.get_person_from_handle(handle)
-            if self.filt.apply(db, person):
-                target_people.append(handle)
+        for person_data in db.iter_raw_person_data():
+            if self.filt.apply_to_one(db, person_data):
+                target_people.append(data["handle"])
             if user:
                 user.step_progress()
         if user:
diff --git a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
index 20cd731760..25a27192f3 100644
--- a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
@@ -34,6 +34,7 @@
 # -------------------------------------------------------------------------
 from .. import Rule
 from ._matchesfilter import MatchesFilter
+from gramps.gen.lib.serialize import from_dict
 
 
 # -------------------------------------------------------------------------
@@ -61,11 +62,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for person_data in db.iter_raw_person():
             if user:
                 user.step_progress()
-            if self.filt.apply(db, person):
-                self.init_list(person)
+            if self.filt.apply_to_one(db, person_data):
+                self.init_list(from_dict(person_data))
         if user:
             user.end_progress()
 
diff --git a/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py b/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
index 25634a1bc1..fa95ceab77 100644
--- a/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
@@ -24,6 +24,7 @@
 #
 # -------------------------------------------------------------------------
 from ....const import GRAMPS_LOCALE as glocale
+from gramps.gen.lib.serialize import from_dict
 
 _ = glocale.translation.gettext
 
@@ -65,11 +66,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for person_data in db.iter_raw_person_data():
             if user:
                 user.step_progress()
-            if self.matchfilt.apply(db, person):
-                self.add_matches(person)
+            if self.matchfilt.apply_to_one(db, person_data):
+                self.add_matches(from_dict(person_data))
         if user:
             user.end_progress()
 
diff --git a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
index 4f3288d996..973cb289d2 100644
--- a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
@@ -24,6 +24,7 @@
 #
 # -------------------------------------------------------------------------
 from ....const import GRAMPS_LOCALE as glocale
+from gramps.gen.lib.serialize import from_dict
 
 _ = glocale.translation.gettext
 
@@ -71,11 +72,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for person_data in db.iter_raw_person_data():
             if user:
                 user.step_progress()
-            if self.filt.apply(db, person):
-                self.init_list(person, first)
+            if self.filt.apply_to_one(db, person_data):
+                self.init_list(from_dict(person_data), first)
         if user:
             user.end_progress()
 
diff --git a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
index 2c7b047df6..631ede7558 100644
--- a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
@@ -34,6 +34,7 @@
 # -------------------------------------------------------------------------
 from .. import Rule
 from ._matchesfilter import MatchesFilter
+from gramps.gen.lib.serialize import from_dict
 
 
 # -------------------------------------------------------------------------
@@ -61,11 +62,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for person_data in db.iter_raw_person_data():
             if user:
                 user.step_progress()
-            if self.filt.apply(db, person):
-                self.init_list(person)
+            if self.filt.apply_to_one(db, person_data):
+                self.init_list(from_dict(person_data))
         if user:
             user.end_progress()
 
diff --git a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
index d770cda16b..8482f761fa 100644
--- a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
@@ -34,6 +34,7 @@
 # -------------------------------------------------------------------------
 from .. import Rule
 from ._matchesfilter import MatchesFilter
+from gramps.gen.lib.serialize import from_dict
 
 
 # -------------------------------------------------------------------------
@@ -60,11 +61,11 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person in db.iter_people():
+        for person_data in db.iter_raw_person_data():
             if user:
                 user.step_progress()
-            if self.matchfilt.apply(db, person):
-                self.init_list(person)
+            if self.matchfilt.apply_to_one(db, person_data):
+                self.init_list(from_dict(person_data))
         if user:
             user.end_progress()
 
diff --git a/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py b/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
index d22cf416c9..04a62740bc 100644
--- a/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isspouseoffiltermatch.py
@@ -67,7 +67,8 @@ def apply_to_one(self, db, data):
                         continue
                     if spouse_id == person.handle:
                         continue
-                    if self.filt.apply(db, db.get_person_from_handle(spouse_id)):
+                    person_data = db.get_raw_person_data(spouse_id)
+                    if self.filt.apply_to_one(db, person_data):
                         return True
         return False
 
diff --git a/gramps/gen/filters/rules/test/event_rules_test.py b/gramps/gen/filters/rules/test/event_rules_test.py
index 380e15cdec..07bd2f7376 100644
--- a/gramps/gen/filters/rules/test/event_rules_test.py
+++ b/gramps/gen/filters/rules/test/event_rules_test.py
@@ -73,7 +73,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericEventFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allevents(self):
diff --git a/gramps/gen/filters/rules/test/family_rules_test.py b/gramps/gen/filters/rules/test/family_rules_test.py
index f9afa9b7b3..987022f859 100644
--- a/gramps/gen/filters/rules/test/family_rules_test.py
+++ b/gramps/gen/filters/rules/test/family_rules_test.py
@@ -84,7 +84,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericFamilyFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allfamilies(self):
diff --git a/gramps/gen/filters/rules/test/media_rules_test.py b/gramps/gen/filters/rules/test/media_rules_test.py
index 10f6111df6..d0c24eb9cf 100644
--- a/gramps/gen/filters/rules/test/media_rules_test.py
+++ b/gramps/gen/filters/rules/test/media_rules_test.py
@@ -70,7 +70,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericMediaFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allmedia(self):
diff --git a/gramps/gen/filters/rules/test/note_rules_test.py b/gramps/gen/filters/rules/test/note_rules_test.py
index ab29ccce9b..f86041d788 100644
--- a/gramps/gen/filters/rules/test/note_rules_test.py
+++ b/gramps/gen/filters/rules/test/note_rules_test.py
@@ -65,7 +65,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericNoteFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allnotes(self):
diff --git a/gramps/gen/filters/rules/test/person_rules_test.py b/gramps/gen/filters/rules/test/person_rules_test.py
index a62beb8940..d3f7fa0542 100644
--- a/gramps/gen/filters/rules/test/person_rules_test.py
+++ b/gramps/gen/filters/rules/test/person_rules_test.py
@@ -147,7 +147,7 @@ def filter_with_rule(
         filter_.set_logical_op(l_op)
         filter_.set_invert(invert)
         stime = perf_counter()
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         # if __debug__:
         # frame = inspect.currentframe()
         # rulename = frame.f_back.f_code.co_name
diff --git a/gramps/gen/filters/rules/test/place_rules_test.py b/gramps/gen/filters/rules/test/place_rules_test.py
index b027cfadce..d3a274c8d2 100644
--- a/gramps/gen/filters/rules/test/place_rules_test.py
+++ b/gramps/gen/filters/rules/test/place_rules_test.py
@@ -76,7 +76,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericPlaceFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allplaces(self):
diff --git a/gramps/gen/filters/rules/test/repository_rules_test.py b/gramps/gen/filters/rules/test/repository_rules_test.py
index fa4495afb7..3f1bb28861 100644
--- a/gramps/gen/filters/rules/test/repository_rules_test.py
+++ b/gramps/gen/filters/rules/test/repository_rules_test.py
@@ -64,7 +64,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericRepositoryFilter()
         filter_.add_rule(rule)
-        results = filter_.apply(self.db)
+        results = filter_.apply_to_all(self.db)
         return set(results)
 
     def test_allrepos(self):

From 03262be1b71dd5258e09dd2dd7734daa3f911fac Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 17 Nov 2024 09:55:28 -0500
Subject: [PATCH 29/46] Clean up; use handles_out to speed-up when no
 handles_in

---
 gramps/gen/filters/_genericfilter.py | 45 +++++++++++++++++-----------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index ae230df903..be9bb1240b 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -145,6 +145,7 @@ def walk_filters(self, filter, parent_invert, result):
         Walk all of the filters/rules and get
         rules with maps
         """
+        current_invert = parent_invert if not filter.invert else not parent_invert
         LOG.debug(
             "walking, filter: %s, invert=%s, parent_invert=%s",
             filter,
@@ -156,7 +157,7 @@ def walk_filters(self, filter, parent_invert, result):
             if hasattr(item, "find_filter"):
                 self.walk_filters(
                     item.find_filter(),
-                    not parent_invert if filter.invert else parent_invert,
+                    current_invert,
                     result,
                 )
             elif hasattr(item, "map"):
@@ -172,7 +173,7 @@ def walk_filters(self, filter, parent_invert, result):
             )
             result.append(
                 (
-                    parent_invert if not filter.invert else not parent_invert,
+                    current_invert,
                     filter.logical_op,
                     rules,
                 )
@@ -209,28 +210,32 @@ def union(sets):
         all_maps = []
         self.walk_filters(self, self.invert, all_maps)
         LOG.debug("number of maps to consider: %s", len(all_maps))
-        handles = None
+        handles_in = None
+        handles_out = set()
         # Get all positive non-inverted maps
         for inverted, logical_op, maps in all_maps:
             if logical_op == "and" and not inverted:
                 LOG.debug("optimizer positive match!")
-                if handles is None:
-                    handles = intersection(maps)
+                if handles_in is None:
+                    handles_in = intersection(maps)
                 else:
-                    handles = intersection([handles] + maps)
-        if handles is not None:
-            # Get all inverted maps and remove from handles
-            for inverted, logical_op, maps in all_maps:
-                if logical_op == "and" and inverted:
-                    LOG.debug("optimizer inverted match!")
-                    not_handles = union(maps)
-                    handles = handles - not_handles
+                    handles_in = intersection([handles_in] + maps)
+
+        # Get all inverted maps:
+        for inverted, logical_op, maps in all_maps:
+            if logical_op == "and" and inverted:
+                LOG.debug("optimizer inverted match!")
+                handles_out = union([handles_out] + maps)
+
+        if handles_in is not None:
+            handles_in = handles_in - handles_out
+
         # ------------------------------------------------------
-        LOG.debug("optimizer handles: %s", len(handles) if handles else 0)
+        LOG.debug("optimizer handles_in: %s", len(handles_in) if handles_in else 0)
         if id_list is None:
-            if handles is not None:
+            if handles_in is not None:
                 # Use these rather than going through entire database
-                for handle in handles:
+                for handle in handles_in:
                     json_data = self.get_raw_data(db, handle)
 
                     if user:
@@ -244,6 +249,9 @@ def union(sets):
                     self.get_tree_cursor(db) if tree else self.get_cursor(db)
                 ) as cursor:
                     for handle, data in cursor:
+                        if handle in handles_out:
+                            continue
+
                         if user:
                             user.step_progress()
 
@@ -257,7 +265,10 @@ def union(sets):
                 else:
                     handle = data[tupleind]
 
-                if handles is not None and handle not in handles:
+                if handles_in is not None:
+                    if handle not in handles_in:
+                        continue
+                elif handle in handles_out:
                     continue
 
                 json_data = self.get_raw_data(db, handle)

From 6094adf091b60fae36a1ef1b413efafbd7b82810 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 17 Nov 2024 10:07:15 -0500
Subject: [PATCH 30/46] Add table name to each generic filter type; remove
 list() from hastag

---
 gramps/gen/filters/_genericfilter.py    | 17 +++++++++++++++++
 gramps/gen/filters/rules/_hastagbase.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index be9bb1240b..0d4f9442df 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -58,6 +58,7 @@ class GenericFilter:
     """Filter class that consists of several rules."""
 
     logical_functions = ["and", "or", "one"]
+    table = "person"
 
     def __init__(self, source=None):
         if source:
@@ -360,6 +361,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericFamilyFilter(GenericFilter):
+    tables = "family"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -380,6 +383,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericEventFilter(GenericFilter):
+    table = "event"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -400,6 +405,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericSourceFilter(GenericFilter):
+    table = "source"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -420,6 +427,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericCitationFilter(GenericFilter):
+    table = "citation"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -443,6 +452,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericPlaceFilter(GenericFilter):
+    table = "place"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -466,6 +477,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericMediaFilter(GenericFilter):
+    table = "media"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -486,6 +499,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericRepoFilter(GenericFilter):
+    table = "repository"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
@@ -506,6 +521,8 @@ def get_raw_data(self, db, handle):
 
 
 class GenericNoteFilter(GenericFilter):
+    table = "note"
+
     def __init__(self, source=None):
         GenericFilter.__init__(self, source)
 
diff --git a/gramps/gen/filters/rules/_hastagbase.py b/gramps/gen/filters/rules/_hastagbase.py
index edc9fbaeeb..ada365691b 100644
--- a/gramps/gen/filters/rules/_hastagbase.py
+++ b/gramps/gen/filters/rules/_hastagbase.py
@@ -66,7 +66,7 @@ def prepare(self, db, user):
                 ["$.handle"],
                 ("$.tag_list", "LIKE", f'%"{self.tag_handle}"%'),
             )
-            self.map = set([row["handle"] for row in list(results)])
+            self.map = set([row["handle"] for row in results])
         else:
             self.map = set()
 

From ec726730aa8bc33d2b733a36bf36067f1900902e Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Sun, 17 Nov 2024 11:12:10 -0500
Subject: [PATCH 31/46] Fixes for tests; yeah tests!

---
 gramps/gen/filters/_genericfilter.py                        | 5 ++++-
 gramps/gen/filters/rules/_hasldsbase.py                     | 2 +-
 gramps/gen/filters/rules/event/_hastype.py                  | 2 +-
 .../filters/rules/person/_deeprelationshippathbetween.py    | 2 +-
 gramps/gen/filters/rules/person/_hascommonancestorwith.py   | 3 +--
 .../rules/person/_hascommonancestorwithfiltermatch.py       | 2 +-
 gramps/gen/filters/rules/person/_isancestoroffiltermatch.py | 4 ++--
 gramps/gen/filters/rules/person/_ischildoffiltermatch.py    | 2 +-
 .../rules/person/_isdescendantfamilyoffiltermatch.py        | 6 +++---
 .../gen/filters/rules/person/_isdescendantoffiltermatch.py  | 2 +-
 gramps/gen/filters/rules/person/_isparentoffiltermatch.py   | 2 +-
 gramps/gen/filters/rules/person/_issiblingoffiltermatch.py  | 2 +-
 12 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index 0d4f9442df..e2add09812 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -237,13 +237,16 @@ def union(sets):
             if handles_in is not None:
                 # Use these rather than going through entire database
                 for handle in handles_in:
+                    if handle is None:
+                        continue
+
                     json_data = self.get_raw_data(db, handle)
 
                     if user:
                         user.step_progress()
 
                     if apply_logical_op(db, json_data, self.flist) != self.invert:
-                        final_list.append(json_data)
+                        final_list.append(json_data["handle"])
 
             else:
                 with (
diff --git a/gramps/gen/filters/rules/_hasldsbase.py b/gramps/gen/filters/rules/_hasldsbase.py
index df9cb79ede..a1662a13ff 100644
--- a/gramps/gen/filters/rules/_hasldsbase.py
+++ b/gramps/gen/filters/rules/_hasldsbase.py
@@ -64,7 +64,7 @@ def prepare(self, db, user):
         self.userSelectedCount = int(self.list[0])
 
     def apply_to_one(self, db, data):
-        obj = self.get_obj(data)
+        obj = self.get_object(data)
         count = len(obj.get_lds_ord_list())
         if self.count_type == 0:  # "less than"
             return count < self.userSelectedCount
diff --git a/gramps/gen/filters/rules/event/_hastype.py b/gramps/gen/filters/rules/event/_hastype.py
index 5e08352fff..3d9630d107 100644
--- a/gramps/gen/filters/rules/event/_hastype.py
+++ b/gramps/gen/filters/rules/event/_hastype.py
@@ -65,7 +65,7 @@ def apply_to_one(self, _db, data):
         """
         Apply the rule. Return True if a match.
         """
-        if data["event_type"]:
+        if data["type"]:
             obj = self.get_object(data)
             # FIXME: probably don't need object to do this:
             return obj.get_type() == self.event_type
diff --git a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
index 938d9cb0c8..ec1ad073fc 100644
--- a/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
+++ b/gramps/gen/filters/rules/person/_deeprelationshippathbetween.py
@@ -157,7 +157,7 @@ def prepare(self, db, user):
                 db.get_number_of_people(),
             )
         target_people = []
-        for person_data in db.iter_raw_person_data():
+        for handle, person_data in db._iter_raw_person_data():
             if self.filt.apply_to_one(db, person_data):
                 target_people.append(data["handle"])
             if user:
diff --git a/gramps/gen/filters/rules/person/_hascommonancestorwith.py b/gramps/gen/filters/rules/person/_hascommonancestorwith.py
index 5b90780b12..de069e81e1 100644
--- a/gramps/gen/filters/rules/person/_hascommonancestorwith.py
+++ b/gramps/gen/filters/rules/person/_hascommonancestorwith.py
@@ -107,8 +107,7 @@ def has_common_ancestor(self, other):
         return False
 
     def apply_to_one(self, db, data):
+        person = self.get_object(data)
         if data["handle"] not in self.ancestor_cache:
-            person = self.get_object(data)
             self.add_ancs(db, person)
-
         return self.has_common_ancestor(person)
diff --git a/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py b/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
index 7aa4c49725..6384b826a0 100644
--- a/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
+++ b/gramps/gen/filters/rules/person/_hascommonancestorwithfiltermatch.py
@@ -74,7 +74,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for data in db.iter_raw_person_data():
+        for handle, data in db._iter_raw_person_data():
             # person = db.get_person_from_handle(handle)
             if user:
                 user.step_progress()
diff --git a/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py b/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
index 74c9fc920f..4d013c700b 100644
--- a/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isancestoroffiltermatch.py
@@ -24,7 +24,7 @@
 #
 # -------------------------------------------------------------------------
 from ....const import GRAMPS_LOCALE as glocale
-from gramps.gen.lib.serialize import to_dict
+from gramps.gen.lib.serialize import from_dict
 
 _ = glocale.translation.gettext
 
@@ -72,7 +72,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for data in db.iter_raw_person_data():
+        for handle, data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.filt.apply_to_one(db, data):
diff --git a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
index 25a27192f3..2eeca007d6 100644
--- a/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_ischildoffiltermatch.py
@@ -62,7 +62,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person_data in db.iter_raw_person():
+        for handle, person_data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.filt.apply_to_one(db, person_data):
diff --git a/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py b/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
index fa95ceab77..19bbf8def7 100644
--- a/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isdescendantfamilyoffiltermatch.py
@@ -56,7 +56,7 @@ class IsDescendantFamilyOfFilterMatch(IsDescendantFamilyOf):
 
     def prepare(self, db, user):
         self.db = db
-        self.matches = set()
+        self.map = set()
 
         self.matchfilt = MatchesFilter(self.list[0:1])
         self.matchfilt.requestprepare(db, user)
@@ -66,7 +66,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person_data in db.iter_raw_person_data():
+        for handle, person_data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.matchfilt.apply_to_one(db, person_data):
@@ -76,4 +76,4 @@ def prepare(self, db, user):
 
     def reset(self):
         self.matchfilt.requestreset()
-        self.matches = set()
+        self.map = set()
diff --git a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
index 973cb289d2..30b5fe2302 100644
--- a/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isdescendantoffiltermatch.py
@@ -72,7 +72,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person_data in db.iter_raw_person_data():
+        for handle, person_data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.filt.apply_to_one(db, person_data):
diff --git a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
index 631ede7558..14bf8dec59 100644
--- a/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_isparentoffiltermatch.py
@@ -62,7 +62,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person_data in db.iter_raw_person_data():
+        for handle, person_data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.filt.apply_to_one(db, person_data):
diff --git a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
index 8482f761fa..344730af5e 100644
--- a/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
+++ b/gramps/gen/filters/rules/person/_issiblingoffiltermatch.py
@@ -61,7 +61,7 @@ def prepare(self, db, user):
                 _("Retrieving all sub-filter matches"),
                 db.get_number_of_people(),
             )
-        for person_data in db.iter_raw_person_data():
+        for handle, person_data in db._iter_raw_person_data():
             if user:
                 user.step_progress()
             if self.matchfilt.apply_to_one(db, person_data):

From e3f74876eddadffde704dfcaf7dd96921391cd1a Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Tue, 19 Nov 2024 16:57:45 -0500
Subject: [PATCH 32/46] Fix logic bug; add preparation progress; progress
 matches actual count

---
 gramps/gen/filters/_genericfilter.py | 39 +++++++++++++++++++---------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index e2add09812..ae64beb4d4 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -184,8 +184,6 @@ def apply_logical_op_to_all(
         self, db, id_list, apply_logical_op, user=None, tupleind=None, tree=False
     ):
         final_list = []
-        if user:
-            user.begin_progress(_("Filter"), _("Applying ..."), self.get_number(db))
 
         def intersection(sets):
             if sets:
@@ -209,7 +207,7 @@ def union(sets):
         # Optimiziation
         # ------------------------------------------------------
         all_maps = []
-        self.walk_filters(self, self.invert, all_maps)
+        self.walk_filters(self, False, all_maps)
         LOG.debug("number of maps to consider: %s", len(all_maps))
         handles_in = None
         handles_out = set()
@@ -235,16 +233,19 @@ def union(sets):
         LOG.debug("optimizer handles_in: %s", len(handles_in) if handles_in else 0)
         if id_list is None:
             if handles_in is not None:
+                if user:
+                    user.begin_progress(_("Filter"), _("Applying ..."), len(handles))
+
                 # Use these rather than going through entire database
                 for handle in handles_in:
+                    if user:
+                        user.step_progress()
+
                     if handle is None:
                         continue
 
                     json_data = self.get_raw_data(db, handle)
 
-                    if user:
-                        user.step_progress()
-
                     if apply_logical_op(db, json_data, self.flist) != self.invert:
                         final_list.append(json_data["handle"])
 
@@ -252,18 +253,28 @@ def union(sets):
                 with (
                     self.get_tree_cursor(db) if tree else self.get_cursor(db)
                 ) as cursor:
-                    for handle, data in cursor:
-                        if handle in handles_out:
-                            continue
+                    if user:
+                        user.begin_progress(
+                            _("Filter"), _("Applying ..."), self.get_number(db)
+                        )
 
+                    for handle, data in cursor:
                         if user:
                             user.step_progress()
 
+                        if handle in handles_out:
+                            continue
+
                         if apply_logical_op(db, data, self.flist) != self.invert:
                             final_list.append(handle)
 
         else:
+            if user:
+                user.begin_progress(_("Filter"), _("Applying ..."), len(id_list))
             for data in id_list:
+                if user:
+                    user.step_progress()
+
                 if tupleind is None:
                     handle = data
                 else:
@@ -277,9 +288,6 @@ def union(sets):
 
                 json_data = self.get_raw_data(db, handle)
 
-                if user:
-                    user.step_progress()
-
                 if apply_logical_op(db, json_data, self.flist) != self.invert:
                     final_list.append(data)
 
@@ -337,9 +345,16 @@ def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
                 if id_list not given, all items in the database that
                 match the filter are returned as a list of handles
         """
+        if user:
+            user.begin_progress(_("Filter"), _("Preparing ..."), len(self.flist))
 
         for rule in self.flist:
             rule.requestprepare(db, user)
+            if user:
+                user.step_progress()
+
+        if user:
+            user.end_progress()
 
         if self.logical_op == "and":
             apply_logical_op = self.and_test

From 69366af6121306449d8b99dbbc60a57a058bb321 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Thu, 21 Nov 2024 11:52:03 -0500
Subject: [PATCH 33/46] No need to rename filter.apply

---
 gramps/gen/filters/_genericfilter.py           |  2 +-
 gramps/gen/filters/_paramfilter.py             |  4 ++--
 .../gen/filters/rules/test/event_rules_test.py |  2 +-
 .../filters/rules/test/family_rules_test.py    |  2 +-
 .../gen/filters/rules/test/media_rules_test.py |  2 +-
 .../gen/filters/rules/test/note_rules_test.py  |  2 +-
 .../filters/rules/test/person_rules_test.py    |  2 +-
 .../gen/filters/rules/test/place_rules_test.py |  2 +-
 .../rules/test/repository_rules_test.py        |  2 +-
 gramps/gen/proxy/filter.py                     |  6 +++---
 gramps/gui/editors/filtereditor.py             |  2 +-
 gramps/gui/views/treemodels/flatbasemodel.py   |  4 ++--
 gramps/gui/views/treemodels/treebasemodel.py   |  2 +-
 gramps/plugins/drawreport/calendarreport.py    |  2 +-
 gramps/plugins/drawreport/statisticschart.py   |  2 +-
 gramps/plugins/drawreport/timeline.py          |  4 ++--
 gramps/plugins/gramplet/todogramplet.py        |  2 +-
 gramps/plugins/graph/gvrelgraph.py             |  2 +-
 gramps/plugins/lib/librecords.py               |  4 ++--
 gramps/plugins/quickview/samesurnames.py       |  4 ++--
 gramps/plugins/textreport/birthdayreport.py    |  2 +-
 gramps/plugins/textreport/familygroup.py       |  2 +-
 gramps/plugins/textreport/indivcomplete.py     |  2 +-
 gramps/plugins/textreport/placereport.py       |  2 +-
 gramps/plugins/textreport/tagreport.py         | 18 +++++++++---------
 gramps/plugins/tool/eventcmp.py                |  2 +-
 gramps/plugins/tool/removeunused.py            |  4 ++--
 gramps/plugins/tool/sortevents.py              |  2 +-
 gramps/plugins/view/geoevents.py               |  2 +-
 gramps/plugins/view/geoplaces.py               |  2 +-
 gramps/plugins/webreport/narrativeweb.py       |  2 +-
 gramps/plugins/webreport/webcal.py             |  2 +-
 32 files changed, 48 insertions(+), 48 deletions(-)

diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py
index ae64beb4d4..62c8f58091 100644
--- a/gramps/gen/filters/_genericfilter.py
+++ b/gramps/gen/filters/_genericfilter.py
@@ -328,7 +328,7 @@ def apply_to_one(self, db, data):
             raise Exception("invalid operator: %r" % self.logical_op)
         return res != self.invert
 
-    def apply_to_all(self, db, id_list=None, tupleind=None, user=None, tree=False):
+    def apply(self, db, id_list=None, tupleind=None, user=None, tree=False):
         """
         Apply the filter using db.
         If id_list given, the handles in id_list are used. If not given
diff --git a/gramps/gen/filters/_paramfilter.py b/gramps/gen/filters/_paramfilter.py
index 50a141390d..ebcd5d2a71 100644
--- a/gramps/gen/filters/_paramfilter.py
+++ b/gramps/gen/filters/_paramfilter.py
@@ -45,7 +45,7 @@ def __init__(self, source=None):
     def set_parameter(self, param):
         self.param_list = [param]
 
-    def apply_to_all(self, db, id_list=None, user=None):
+    def apply(self, db, id_list=None, user=None):
         for rule in self.flist:
             # rule.set_list(self.param_list)
             #
@@ -62,7 +62,7 @@ def apply_to_all(self, db, id_list=None, user=None):
                     "Custom filters can not twice be used" " in a parameter filter"
                 )
             rule.requestprepare(db, user)
-        result = GenericFilter.apply_to_all(self, db, id_list)
+        result = GenericFilter.apply(self, db, id_list)
         for rule in self.flist:
             rule.requestreset()
         return result
diff --git a/gramps/gen/filters/rules/test/event_rules_test.py b/gramps/gen/filters/rules/test/event_rules_test.py
index 07bd2f7376..380e15cdec 100644
--- a/gramps/gen/filters/rules/test/event_rules_test.py
+++ b/gramps/gen/filters/rules/test/event_rules_test.py
@@ -73,7 +73,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericEventFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allevents(self):
diff --git a/gramps/gen/filters/rules/test/family_rules_test.py b/gramps/gen/filters/rules/test/family_rules_test.py
index 987022f859..f9afa9b7b3 100644
--- a/gramps/gen/filters/rules/test/family_rules_test.py
+++ b/gramps/gen/filters/rules/test/family_rules_test.py
@@ -84,7 +84,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericFamilyFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allfamilies(self):
diff --git a/gramps/gen/filters/rules/test/media_rules_test.py b/gramps/gen/filters/rules/test/media_rules_test.py
index d0c24eb9cf..10f6111df6 100644
--- a/gramps/gen/filters/rules/test/media_rules_test.py
+++ b/gramps/gen/filters/rules/test/media_rules_test.py
@@ -70,7 +70,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericMediaFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allmedia(self):
diff --git a/gramps/gen/filters/rules/test/note_rules_test.py b/gramps/gen/filters/rules/test/note_rules_test.py
index f86041d788..ab29ccce9b 100644
--- a/gramps/gen/filters/rules/test/note_rules_test.py
+++ b/gramps/gen/filters/rules/test/note_rules_test.py
@@ -65,7 +65,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericNoteFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allnotes(self):
diff --git a/gramps/gen/filters/rules/test/person_rules_test.py b/gramps/gen/filters/rules/test/person_rules_test.py
index d3f7fa0542..a62beb8940 100644
--- a/gramps/gen/filters/rules/test/person_rules_test.py
+++ b/gramps/gen/filters/rules/test/person_rules_test.py
@@ -147,7 +147,7 @@ def filter_with_rule(
         filter_.set_logical_op(l_op)
         filter_.set_invert(invert)
         stime = perf_counter()
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         # if __debug__:
         # frame = inspect.currentframe()
         # rulename = frame.f_back.f_code.co_name
diff --git a/gramps/gen/filters/rules/test/place_rules_test.py b/gramps/gen/filters/rules/test/place_rules_test.py
index d3a274c8d2..b027cfadce 100644
--- a/gramps/gen/filters/rules/test/place_rules_test.py
+++ b/gramps/gen/filters/rules/test/place_rules_test.py
@@ -76,7 +76,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericPlaceFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allplaces(self):
diff --git a/gramps/gen/filters/rules/test/repository_rules_test.py b/gramps/gen/filters/rules/test/repository_rules_test.py
index 3f1bb28861..fa4495afb7 100644
--- a/gramps/gen/filters/rules/test/repository_rules_test.py
+++ b/gramps/gen/filters/rules/test/repository_rules_test.py
@@ -64,7 +64,7 @@ def filter_with_rule(self, rule):
         """
         filter_ = GenericRepositoryFilter()
         filter_.add_rule(rule)
-        results = filter_.apply_to_all(self.db)
+        results = filter_.apply(self.db)
         return set(results)
 
     def test_allrepos(self):
diff --git a/gramps/gen/proxy/filter.py b/gramps/gen/proxy/filter.py
index 5414e3fcfc..8c172de1bc 100644
--- a/gramps/gen/proxy/filter.py
+++ b/gramps/gen/proxy/filter.py
@@ -67,7 +67,7 @@ def __init__(
         self.person_filter = person_filter
         if person_filter:
             self.plist = set(
-                person_filter.apply_to_all(
+                person_filter.apply(
                     self.db, self.db.iter_person_handles(), user=user
                 )
             )
@@ -76,7 +76,7 @@ def __init__(
 
         if event_filter:
             self.elist = set(
-                event_filter.apply_to_all(
+                event_filter.apply(
                     self.db, self.db.iter_event_handles(), user=user
                 )
             )
@@ -85,7 +85,7 @@ def __init__(
 
         if note_filter:
             self.nlist = set(
-                note_filter.apply_to_all(
+                note_filter.apply(
                     self.db, self.db.iter_note_handles(), user=user
                 )
             )
diff --git a/gramps/gui/editors/filtereditor.py b/gramps/gui/editors/filtereditor.py
index 293810d236..ae6eaf6970 100644
--- a/gramps/gui/editors/filtereditor.py
+++ b/gramps/gui/editors/filtereditor.py
@@ -1312,7 +1312,7 @@ def test_clicked(self, obj):
         if node:
             filt = self.clist.get_object(node)
             try:
-                handle_list = filt.apply_to_all(self.db, self.get_all_handles())
+                handle_list = filt.apply(self.db, self.get_all_handles())
             except FilterError as msg:
                 (msg1, msg2) = msg.messages()
                 ErrorDialog(msg1, msg2, parent=self.window)
diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py
index 5f7c1e1b93..97eeb5e73c 100644
--- a/gramps/gui/views/treemodels/flatbasemodel.py
+++ b/gramps/gui/views/treemodels/flatbasemodel.py
@@ -627,11 +627,11 @@ def _rebuild_filter(self, ignore=None):
             if self.search:
                 ident = False
                 if ignore is None:
-                    dlist = self.search.apply_to_all(
+                    dlist = self.search.apply(
                         cdb, allkeys, tupleind=1, user=self.user
                     )
                 else:
-                    dlist = self.search.apply_to_all(
+                    dlist = self.search.apply(
                         cdb, [k for k in allkeys if k[1] != ignore], tupleind=1
                     )
             elif ignore is None:
diff --git a/gramps/gui/views/treemodels/treebasemodel.py b/gramps/gui/views/treemodels/treebasemodel.py
index bfdafc956c..a217c46698 100644
--- a/gramps/gui/views/treemodels/treebasemodel.py
+++ b/gramps/gui/views/treemodels/treebasemodel.py
@@ -623,7 +623,7 @@ def __rebuild_filter(self, dfilter, skip, items, gen_cursor, data_map, add_func)
         assert not skip
         if dfilter:
             cdb = CacheProxyDb(self.db)
-            for handle in dfilter.apply_to_all(
+            for handle in dfilter.apply(
                 cdb,
                 tree=True,
                 user=User(parent=self.uistate.window, uistate=self.uistate),
diff --git a/gramps/plugins/drawreport/calendarreport.py b/gramps/plugins/drawreport/calendarreport.py
index 3d99efe37f..f928e44bc2 100644
--- a/gramps/plugins/drawreport/calendarreport.py
+++ b/gramps/plugins/drawreport/calendarreport.py
@@ -351,7 +351,7 @@ def collect_data(self):
         and text.
         """
         people = self.database.iter_person_handles()
-        people = self.filter.apply_to_all(self.database, people, user=self._user)
+        people = self.filter.apply(self.database, people, user=self._user)
 
         with self._user.progress(
             _("Calendar Report"), _("Reading database..."), len(people)
diff --git a/gramps/plugins/drawreport/statisticschart.py b/gramps/plugins/drawreport/statisticschart.py
index fb0235d34a..6e9ebac21f 100644
--- a/gramps/plugins/drawreport/statisticschart.py
+++ b/gramps/plugins/drawreport/statisticschart.py
@@ -907,7 +907,7 @@ def __init__(self, database, options, user):
             self._get_date(Date(year_to)),
         )
 
-        people = self.filter.apply_to_all(
+        people = self.filter.apply(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
diff --git a/gramps/plugins/drawreport/timeline.py b/gramps/plugins/drawreport/timeline.py
index 5b79591e8e..92d8c7f1c8 100644
--- a/gramps/plugins/drawreport/timeline.py
+++ b/gramps/plugins/drawreport/timeline.py
@@ -142,7 +142,7 @@ def __init__(self, database, options, user):
 
     def write_report(self):
         # Apply the filter
-        self.plist = self.filter.apply_to_all(
+        self.plist = self.filter.apply(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
@@ -385,7 +385,7 @@ def min_max_year(low, high, year):
 
     def name_size(self):
         """get the length of the name"""
-        self.plist = self.filter.apply_to_all(
+        self.plist = self.filter.apply(
             self.database, self.database.iter_person_handles(), user=self._user
         )
 
diff --git a/gramps/plugins/gramplet/todogramplet.py b/gramps/plugins/gramplet/todogramplet.py
index c768bea6b5..d51110e177 100644
--- a/gramps/plugins/gramplet/todogramplet.py
+++ b/gramps/plugins/gramplet/todogramplet.py
@@ -104,7 +104,7 @@ def get_note_list(self):
         FilterClass = GenericFilterFactory("Note")
         filter = FilterClass()
         filter.add_rule(rules.note.HasType(["To Do"]))
-        note_list = filter.apply_to_all(self.dbstate.db, all_notes)
+        note_list = filter.apply(self.dbstate.db, all_notes)
         return note_list
 
     def get_notes(self):
diff --git a/gramps/plugins/graph/gvrelgraph.py b/gramps/plugins/graph/gvrelgraph.py
index 39ecc2919a..567b2411ed 100644
--- a/gramps/plugins/graph/gvrelgraph.py
+++ b/gramps/plugins/graph/gvrelgraph.py
@@ -211,7 +211,7 @@ def __init__(self, database, options, user):
             self.advrelinfo = False
 
     def write_report(self):
-        person_handles = self._filter.apply_to_all(
+        person_handles = self._filter.apply(
             self._db, self._db.iter_person_handles(), user=self._user
         )
         # Hash people in a dictionary for faster inclusion checking
diff --git a/gramps/plugins/lib/librecords.py b/gramps/plugins/lib/librecords.py
index cc192ccda4..e28cf307fd 100644
--- a/gramps/plugins/lib/librecords.py
+++ b/gramps/plugins/lib/librecords.py
@@ -169,7 +169,7 @@ def get_unfiltered_person_from_handle(person_handle):
     person_handle_list = list(person_handle_list)
 
     if filter:
-        person_handle_list = filter.apply_to_all(db, person_handle_list, user=user)
+        person_handle_list = filter.apply(db, person_handle_list, user=user)
 
     for person_handle in person_handle_list:
         person = db.get_person_from_handle(person_handle)
@@ -387,7 +387,7 @@ def get_unfiltered_person_from_handle(person_handle):
         if filter:
             # we don't want many progress reports popping up, so no user=user
             # FIXME:
-            if not filter.apply_to_all(db, [father_handle, mother_handle]):
+            if not filter.apply(db, [father_handle, mother_handle]):
                 continue
 
         father = db.get_person_from_handle(father_handle)
diff --git a/gramps/plugins/quickview/samesurnames.py b/gramps/plugins/quickview/samesurnames.py
index 1b3c36c3fe..e8c3420b78 100644
--- a/gramps/plugins/quickview/samesurnames.py
+++ b/gramps/plugins/quickview/samesurnames.py
@@ -133,7 +133,7 @@ def run(database, document, person):
     else:
         rule = IncompleteSurname([])
     filter.add_rule(rule)
-    people = filter.apply_to_all(database, database.iter_person_handles())
+    people = filter.apply(database, database.iter_person_handles())
 
     matches = 0
     for person_handle in people:
@@ -183,7 +183,7 @@ def run_given(database, document, person):
     else:
         rule = IncompleteGiven([])
     filter.add_rule(rule)
-    people = filter.apply_to_all(database, database.iter_person_handles())
+    people = filter.apply(database, database.iter_person_handles())
 
     matches = 0
     for person_handle in people:
diff --git a/gramps/plugins/textreport/birthdayreport.py b/gramps/plugins/textreport/birthdayreport.py
index 8eb4c4bdcc..e0d10eef73 100644
--- a/gramps/plugins/textreport/birthdayreport.py
+++ b/gramps/plugins/textreport/birthdayreport.py
@@ -291,7 +291,7 @@ def collect_data(self):
         and text.
         """
         people = self.database.iter_person_handles()
-        people = self.filter.apply_to_all(self.database, people, user=self._user)
+        people = self.filter.apply(self.database, people, user=self._user)
 
         ngettext = self._locale.translation.ngettext  # to see "nearby" comments
         rel_calc = get_relationship_calculator(reinit=True, clocale=self._locale)
diff --git a/gramps/plugins/textreport/familygroup.py b/gramps/plugins/textreport/familygroup.py
index ed445ffcce..a521a13af8 100644
--- a/gramps/plugins/textreport/familygroup.py
+++ b/gramps/plugins/textreport/familygroup.py
@@ -677,7 +677,7 @@ def write_report(self):
         if not self.filter:
             fam_list = flist
         else:
-            fam_list = self.filter.apply_to_all(self.db, flist, user=self._user)
+            fam_list = self.filter.apply(self.db, flist, user=self._user)
         if fam_list:
             with self._user.progress(
                 _("Family Group Report"), _("Writing families"), len(fam_list)
diff --git a/gramps/plugins/textreport/indivcomplete.py b/gramps/plugins/textreport/indivcomplete.py
index d1120e6a24..57676585d6 100644
--- a/gramps/plugins/textreport/indivcomplete.py
+++ b/gramps/plugins/textreport/indivcomplete.py
@@ -850,7 +850,7 @@ def write_report(self):
         """write the report"""
         plist = self._db.get_person_handles(sort_handles=True, locale=self._locale)
         if self.filter:
-            ind_list = self.filter.apply_to_all(self._db, plist, user=self._user)
+            ind_list = self.filter.apply(self._db, plist, user=self._user)
         else:
             ind_list = plist
         if not ind_list:
diff --git a/gramps/plugins/textreport/placereport.py b/gramps/plugins/textreport/placereport.py
index 147b42990c..9f13b7ca33 100644
--- a/gramps/plugins/textreport/placereport.py
+++ b/gramps/plugins/textreport/placereport.py
@@ -125,7 +125,7 @@ def __init__(self, database, options, user):
         if self.filter.get_name() != "":
             # Use the selected filter to provide a list of place handles
             plist = self._db.iter_place_handles()
-            self.place_handles = self.filter.apply_to_all(
+            self.place_handles = self.filter.apply(
                 self._db, plist, user=self._user
             )
 
diff --git a/gramps/plugins/textreport/tagreport.py b/gramps/plugins/textreport/tagreport.py
index 6592f60a10..a5af6ff737 100644
--- a/gramps/plugins/textreport/tagreport.py
+++ b/gramps/plugins/textreport/tagreport.py
@@ -147,7 +147,7 @@ def write_people(self):
         filter_class = GenericFilterFactory("Person")
         a_filter = filter_class()
         a_filter.add_rule(rules.person.HasTag([self.tag]))
-        ind_list = a_filter.apply_to_all(self.database, plist)
+        ind_list = a_filter.apply(self.database, plist)
 
         if not ind_list:
             return
@@ -235,7 +235,7 @@ def write_families(self):
         filter_class = GenericFilterFactory("Family")
         a_filter = filter_class()
         a_filter.add_rule(rules.family.HasTag([self.tag]))
-        fam_list = a_filter.apply_to_all(self.database, flist)
+        fam_list = a_filter.apply(self.database, flist)
 
         if not fam_list:
             return
@@ -324,7 +324,7 @@ def write_events(self):
         filter_class = GenericFilterFactory("Event")
         a_filter = filter_class()
         a_filter.add_rule(rules.event.HasTag([self.tag]))
-        event_list = a_filter.apply_to_all(self.database, elist)
+        event_list = a_filter.apply(self.database, elist)
 
         if not event_list:
             return
@@ -406,7 +406,7 @@ def write_places(self):
         filter_class = GenericFilterFactory("Place")
         a_filter = filter_class()
         a_filter.add_rule(rules.place.HasTag([self.tag]))
-        place_list = a_filter.apply_to_all(self.database, plist)
+        place_list = a_filter.apply(self.database, plist)
 
         if not place_list:
             return
@@ -487,7 +487,7 @@ def write_notes(self):
         filter_class = GenericFilterFactory("Note")
         a_filter = filter_class()
         a_filter.add_rule(rules.note.HasTag([self.tag]))
-        note_list = a_filter.apply_to_all(self.database, nlist)
+        note_list = a_filter.apply(self.database, nlist)
 
         if not note_list:
             return
@@ -559,7 +559,7 @@ def write_media(self):
         filter_class = GenericFilterFactory("Media")
         a_filter = filter_class()
         a_filter.add_rule(rules.media.HasTag([self.tag]))
-        media_list = a_filter.apply_to_all(self.database, mlist)
+        media_list = a_filter.apply(self.database, mlist)
 
         if not media_list:
             return
@@ -643,7 +643,7 @@ def write_repositories(self):
         filter_class = GenericFilterFactory("Repository")
         a_filter = filter_class()
         a_filter.add_rule(rules.repository.HasTag([self.tag]))
-        repo_list = a_filter.apply_to_all(self.database, rlist)
+        repo_list = a_filter.apply(self.database, rlist)
 
         if not repo_list:
             return
@@ -728,7 +728,7 @@ def write_sources(self):
         filter_class = GenericFilterFactory("Source")
         a_filter = filter_class()
         a_filter.add_rule(rules.source.HasTag([self.tag]))
-        source_list = a_filter.apply_to_all(self.database, slist)
+        source_list = a_filter.apply(self.database, slist)
 
         if not source_list:
             return
@@ -810,7 +810,7 @@ def write_citations(self):
         filter_class = GenericFilterFactory("Citation")
         a_filter = filter_class()
         a_filter.add_rule(rules.citation.HasTag([self.tag]))
-        citation_list = a_filter.apply_to_all(self.database, clist)
+        citation_list = a_filter.apply(self.database, clist)
 
         if not citation_list:
             return
diff --git a/gramps/plugins/tool/eventcmp.py b/gramps/plugins/tool/eventcmp.py
index 69482abc86..fff023f2a6 100644
--- a/gramps/plugins/tool/eventcmp.py
+++ b/gramps/plugins/tool/eventcmp.py
@@ -184,7 +184,7 @@ def on_apply_clicked(self, obj):
         progress_bar = ProgressMeter(_("Comparing events"), "", parent=self.window)
         progress_bar.set_pass(_("Selecting people"), 1)
 
-        plist = cfilter.apply_to_all(self.db, self.db.iter_person_handles())
+        plist = cfilter.apply(self.db, self.db.iter_person_handles())
 
         progress_bar.step()
         progress_bar.close()
diff --git a/gramps/plugins/tool/removeunused.py b/gramps/plugins/tool/removeunused.py
index a3fdecc8fc..c687f60263 100644
--- a/gramps/plugins/tool/removeunused.py
+++ b/gramps/plugins/tool/removeunused.py
@@ -305,10 +305,10 @@ def collect_unused(self):
         FilterClass = GenericFilterFactory("Note")
         filter1 = FilterClass()
         filter1.add_rule(rules.note.HasType(["To Do"]))
-        todo_list = filter1.apply_to_all(self.dbstate.db, all_notes)
+        todo_list = filter1.apply(self.dbstate.db, all_notes)
         filter2 = FilterClass()
         filter2.add_rule(rules.note.HasType(["Link"]))
-        link_list = filter2.apply_to_all(self.dbstate.db, all_notes)
+        link_list = filter2.apply(self.dbstate.db, all_notes)
 
         for the_type, cursor_func, total_func in tables:
             if not self.options.handler.options_dict[the_type]:
diff --git a/gramps/plugins/tool/sortevents.py b/gramps/plugins/tool/sortevents.py
index 55bee61052..3a939f001a 100644
--- a/gramps/plugins/tool/sortevents.py
+++ b/gramps/plugins/tool/sortevents.py
@@ -115,7 +115,7 @@ def sort_person_events(self, trans):
         """
         Sort the personal events associated with the selected people.
         """
-        people_handles = self.filter.apply_to_all(
+        people_handles = self.filter.apply(
             self.db, self.db.iter_person_handles(), user=self._user
         )
         self.progress.set_pass(
diff --git a/gramps/plugins/view/geoevents.py b/gramps/plugins/view/geoevents.py
index 939574cedb..565458d87e 100644
--- a/gramps/plugins/view/geoevents.py
+++ b/gramps/plugins/view/geoevents.py
@@ -371,7 +371,7 @@ def _createmap(self, obj):
             progress.close()
         elif self.generic_filter:
             user = self.uistate.viewmanager.user
-            events_list = self.generic_filter.apply_to_all(dbstate.db, user=user)
+            events_list = self.generic_filter.apply(dbstate.db, user=user)
             progress = ProgressMeter(
                 self.window_name, can_cancel=False, parent=self.uistate.window
             )
diff --git a/gramps/plugins/view/geoplaces.py b/gramps/plugins/view/geoplaces.py
index a74dc503ff..6fd9e94ea3 100644
--- a/gramps/plugins/view/geoplaces.py
+++ b/gramps/plugins/view/geoplaces.py
@@ -422,7 +422,7 @@ def _createmap(self, place_x):
             progress.close()
         elif self.generic_filter:
             user = self.uistate.viewmanager.user
-            place_list = self.generic_filter.apply_to_all(dbstate.db, user=user)
+            place_list = self.generic_filter.apply(dbstate.db, user=user)
             progress = ProgressMeter(
                 self.window_name, can_cancel=False, parent=self.uistate.window
             )
diff --git a/gramps/plugins/webreport/narrativeweb.py b/gramps/plugins/webreport/narrativeweb.py
index ae427ad8e5..0f33138b49 100644
--- a/gramps/plugins/webreport/narrativeweb.py
+++ b/gramps/plugins/webreport/narrativeweb.py
@@ -643,7 +643,7 @@ def _build_obj_dict(self):
             self.obj_dict[obj_class] = defaultdict(set)
 
         ind_list = self._db.iter_person_handles()
-        ind_list = self.filter.apply_to_all(self._db, ind_list, user=self.user)
+        ind_list = self.filter.apply(self._db, ind_list, user=self.user)
 
         message = _("Constructing list of other objects...")
         pgr_title = self.pgrs_title(None)
diff --git a/gramps/plugins/webreport/webcal.py b/gramps/plugins/webreport/webcal.py
index 62a2d0fa6e..d9170e5cd0 100644
--- a/gramps/plugins/webreport/webcal.py
+++ b/gramps/plugins/webreport/webcal.py
@@ -1412,7 +1412,7 @@ def collect_data(self, this_year):
         db = self.database
 
         people = db.iter_person_handles()
-        people = self.filter.apply_to_all(db, people, user=self._user)
+        people = self.filter.apply(db, people, user=self._user)
 
         with self._user.progress(
             _("Web Calendar Report"), _("Reading database..."), len(people)

From 0387e499091a07e5a69ef3b0b42d57f3a99672c8 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Thu, 21 Nov 2024 12:02:40 -0500
Subject: [PATCH 34/46] Linting

---
 gramps/gen/proxy/filter.py                   | 12 +++---------
 gramps/gui/editors/filtereditor.py           |  6 +++++-
 gramps/gui/views/treemodels/flatbasemodel.py |  4 +---
 gramps/plugins/textreport/placereport.py     |  4 +---
 4 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/gramps/gen/proxy/filter.py b/gramps/gen/proxy/filter.py
index 8c172de1bc..d08f558113 100644
--- a/gramps/gen/proxy/filter.py
+++ b/gramps/gen/proxy/filter.py
@@ -67,27 +67,21 @@ def __init__(
         self.person_filter = person_filter
         if person_filter:
             self.plist = set(
-                person_filter.apply(
-                    self.db, self.db.iter_person_handles(), user=user
-                )
+                person_filter.apply(self.db, self.db.iter_person_handles(), user=user)
             )
         else:
             self.plist = set(self.db.iter_person_handles())
 
         if event_filter:
             self.elist = set(
-                event_filter.apply(
-                    self.db, self.db.iter_event_handles(), user=user
-                )
+                event_filter.apply(self.db, self.db.iter_event_handles(), user=user)
             )
         else:
             self.elist = set(self.db.iter_event_handles())
 
         if note_filter:
             self.nlist = set(
-                note_filter.apply(
-                    self.db, self.db.iter_note_handles(), user=user
-                )
+                note_filter.apply(self.db, self.db.iter_note_handles(), user=user)
             )
         else:
             self.nlist = set(self.db.iter_note_handles())
diff --git a/gramps/gui/editors/filtereditor.py b/gramps/gui/editors/filtereditor.py
index ae6eaf6970..84c25878d3 100644
--- a/gramps/gui/editors/filtereditor.py
+++ b/gramps/gui/editors/filtereditor.py
@@ -997,7 +997,11 @@ def on_ok_clicked(self, obj):
         op = (
             "and"
             if val == 0
-            else "or" if val == 1 else "one" if val == 2 else "sequence"
+            else "or"
+            if val == 1
+            else "one"
+            if val == 2
+            else "sequence"
         )
         self.logical.set_active(val)
         self.filter.set_logical_op(op)
diff --git a/gramps/gui/views/treemodels/flatbasemodel.py b/gramps/gui/views/treemodels/flatbasemodel.py
index 97eeb5e73c..41ab69b7bd 100644
--- a/gramps/gui/views/treemodels/flatbasemodel.py
+++ b/gramps/gui/views/treemodels/flatbasemodel.py
@@ -627,9 +627,7 @@ def _rebuild_filter(self, ignore=None):
             if self.search:
                 ident = False
                 if ignore is None:
-                    dlist = self.search.apply(
-                        cdb, allkeys, tupleind=1, user=self.user
-                    )
+                    dlist = self.search.apply(cdb, allkeys, tupleind=1, user=self.user)
                 else:
                     dlist = self.search.apply(
                         cdb, [k for k in allkeys if k[1] != ignore], tupleind=1
diff --git a/gramps/plugins/textreport/placereport.py b/gramps/plugins/textreport/placereport.py
index 9f13b7ca33..308e407d07 100644
--- a/gramps/plugins/textreport/placereport.py
+++ b/gramps/plugins/textreport/placereport.py
@@ -125,9 +125,7 @@ def __init__(self, database, options, user):
         if self.filter.get_name() != "":
             # Use the selected filter to provide a list of place handles
             plist = self._db.iter_place_handles()
-            self.place_handles = self.filter.apply(
-                self._db, plist, user=self._user
-            )
+            self.place_handles = self.filter.apply(self._db, plist, user=self._user)
 
         if places:
             # Add places selected individually

From eff546b6f5e7e418e818953a2dde4e976d0fc770 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Thu, 21 Nov 2024 12:07:58 -0500
Subject: [PATCH 35/46] Fixes from big conversion; linting

---
 gramps/gui/editors/filtereditor.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/gramps/gui/editors/filtereditor.py b/gramps/gui/editors/filtereditor.py
index 84c25878d3..ae6eaf6970 100644
--- a/gramps/gui/editors/filtereditor.py
+++ b/gramps/gui/editors/filtereditor.py
@@ -997,11 +997,7 @@ def on_ok_clicked(self, obj):
         op = (
             "and"
             if val == 0
-            else "or"
-            if val == 1
-            else "one"
-            if val == 2
-            else "sequence"
+            else "or" if val == 1 else "one" if val == 2 else "sequence"
         )
         self.logical.set_active(val)
         self.filter.set_logical_op(op)

From b869070673a65236eb9dde02f22b5fbb38615018 Mon Sep 17 00:00:00 2001
From: Douglas Blank <doug.blank@gmail.com>
Date: Fri, 15 Nov 2024 19:06:54 -0600
Subject: [PATCH 36/46] Update gramps/gen/db/generic.py

Co-authored-by: stevenyoungs <steve@youngs.cc>
---
 gramps/gen/db/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index b1ad404842..d1949fbb4a 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -625,7 +625,7 @@ def _initialize(self, directory, username, password):
 
     def upgrade_table_for_json_data(self, table_name):
         """
-        Overload this method to add JSON access
+        Overload this method to upgrade the table to store data in JSON format
         """
         raise NotImplementedError
 

From 10ef843b71c5119d431bca9f308878ae2ec8c5cd Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 11:51:53 -0500
Subject: [PATCH 37/46] First pass at new conversion

---
 gramps/gen/db/conversion_tools.py | 514 ++++++++++++++++++++++++++++++
 gramps/gen/db/upgrade.py          |  22 +-
 gramps/gen/lib/surname.py         |  19 ++
 3 files changed, 548 insertions(+), 7 deletions(-)
 create mode 100644 gramps/gen/db/conversion_tools.py

diff --git a/gramps/gen/db/conversion_tools.py b/gramps/gen/db/conversion_tools.py
new file mode 100644
index 0000000000..033ebf3e87
--- /dev/null
+++ b/gramps/gen/db/conversion_tools.py
@@ -0,0 +1,514 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2024 Doug Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+
+def convert_21(classname, array):
+    if classname == "Person":
+        return convert_person(array)
+    elif classname == "Family":
+        return convert_family(array)
+    elif classname == "Event":
+        return convert_event(array)
+    elif classname == "Place":
+        return convert_place(array)
+    elif classname == "Repository":
+        return convert_repository(array)
+    elif classname == "Source":
+        return convert_source(array)
+    elif classname == "Citation":
+        return convert_citation(array)
+    elif classname == "Media":
+        return convert_media(array)
+    elif classname == "Note":
+        return convert_note(array)
+    elif classname == "Tag":
+        return convert_tag(array)
+    elif classname == "metadata":
+        value = array
+        type_name = type(value).__name__
+        if type_name in ("set", "tuple"):
+            value = list(value)
+        elif type_name == "Researcher":
+            value = convert_researcher(value)
+        elif type_name in ("int", "str", "list"):
+            pass
+        else:
+            raise Exception("metadata unknown type: %s" % type_name)
+
+        data = {
+            "type": type_name,
+            "value": value,
+        }
+        return data
+
+    else:
+        raise Exception("unknown class: %s" % classname)
+
+
+def convert_researcher(researcher):
+    # This can be a problem if the Researcher class
+    # changes!
+    return {
+        "_class": "Researcher",
+        "street": researcher.street,
+        "city": researcher.city,
+        "county": researcher.county,
+        "state": researcher.state,
+        "country": researcher.country,
+        "postal": researcher.postal,
+        "phone": researcher.phone,
+        "locality": researcher.locality,
+        "name": researcher.name,
+        "addr": researcher.addr,
+        "email": researcher.email,
+    }
+
+
+def convert_person(array):
+    return {
+        "_class": "Person",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "gender": array[2],
+        "primary_name": convert_name(array[3]),
+        "alternate_names": [convert_name(name) for name in array[4]],
+        "death_ref_index": array[5],
+        "birth_ref_index": array[6],
+        "event_ref_list": [convert_event_ref(ref) for ref in array[7]],
+        "family_list": array[8],
+        "parent_family_list": array[9],
+        "media_list": [convert_media_ref(ref) for ref in array[10]],
+        "address_list": [convert_address(address) for address in array[11]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[12]],
+        "urls": [convert_url(url) for url in array[13]],
+        "lds_ord_list": [convert_ord(ord) for ord in array[14]],
+        "citation_list": array[15],  # handles
+        "note_list": array[16],  # handles
+        "change": array[17],
+        "tag_list": array[18],  # handles
+        "private": array[19],
+        "person_ref_list": [convert_person_ref(ref) for ref in array[20]],
+    }
+
+
+def convert_person_ref(array):
+    return {
+        "_class": "PersonRef",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "ref": array[3],
+        "rel": array[4],
+    }
+
+
+def convert_ord(array):
+    return {
+        "_class": "LdsOrd",
+        "citation_list": array[0],  # handles
+        "note_list": array[1],  # handles
+        "date": convert_date(array[2]),
+        "type": array[3],  # string
+        "place": array[4],
+        "famc": array[5],
+        "temple": array[6],
+        "status": array[7],
+        "private": array[8],
+    }
+
+
+def convert_url(array):
+    return {
+        "_class": "Url",
+        "private": array[0],
+        "path": array[1],
+        "desc": array[2],
+        "type": convert_type("UrlType", array[3]),
+    }
+
+
+def convert_address(array):
+    return {
+        "_class": "Address",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "date": convert_date(array[3]),
+        "location": convert_location(array[4]),
+    }
+
+
+def convert_location(array):
+    return {
+        "_class": "Location",
+        "street": array[0],
+        "locality": array[1],
+        "city": array[2],
+        "county": array[3],
+        "state": array[4],
+        "country": array[5],
+        "postal": array[6],
+        "phone": array[7],
+    }
+
+
+def convert_media_ref(array):
+    return {
+        "_class": "MediaRef",
+        "privacy": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
+        "ref": array[4],
+        "rect": list(array[5]) if array[5] is not None else None,
+    }
+
+
+def convert_event_ref(array):
+    return {
+        "_class": "EventRef",
+        "privacy": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
+        "ref": array[4],
+        "role": convert_type("EventRoleType", array[5]),
+    }
+
+
+def convert_attribute(classname, array):
+    if classname == "SrcAttribute":
+        return {
+            "_class": classname,
+            "private": array[0],
+            "type": convert_type("SrcAttributeType", array[1]),
+            "value": array[2],
+        }
+    else:
+        return {
+            "_class": classname,
+            "private": array[0],
+            "citation_list": array[1],  # handles
+            "note_list": array[2],  # handles
+            "type": convert_type("AttributeType", array[3]),
+            "value": array[4],
+        }
+
+
+def convert_name(array):
+    return {
+        "_class": "Name",
+        "private": array[0],
+        "citation_list": array[1],
+        "note": array[2],
+        "date": convert_date(array[3]),
+        "first_name": array[4],
+        "surname_list": [convert_surname(name) for name in array[5]],
+        "suffix": array[6],
+        "title": array[7],
+        "type": convert_type("Note", array[8]),
+        "group_as": array[9],
+        "sort_as": array[10],
+        "display_as": array[11],
+        "call": array[12],
+        "nick": array[13],
+        "famnick": array[14],
+    }
+
+
+def convert_surname(array):
+    return {
+        "_class": "Surname",
+        "surname": array[0],
+        "prefix": array[1],
+        "primary": array[2],
+        "origin_type": convert_type("NameOriginType", array[3]),
+        "connector": array[4],
+    }
+
+
+def convert_date(array):
+    if array is None:
+        return {
+            "_class": "Date",
+            "calendar": 0,
+            "modifier": 0,
+            "quality": 0,
+            "dateval": [0, 0, 0, False],
+            "text": "",
+            "sortval": 0,
+            "newyear": 0,
+        }
+    else:
+        return {
+            "_class": "Date",
+            "calendar": array[0],
+            "modifier": array[1],
+            "quality": array[2],
+            "dateval": list(array[3]),
+            "text": array[4],
+            "sortval": array[5],
+            "newyear": array[6],
+        }
+
+
+def convert_stt(array):
+    return {
+        "_class": "StyledTextTag",
+        "name": convert_type("StyledTextTagType", array[0]),
+        "value": array[1],
+        "ranges": [list(r) for r in array[2]],
+    }
+
+
+def convert_type(classname, array):
+    return {
+        "_class": classname,
+        "value": array[0],
+        "string": array[1],
+    }
+
+
+def convert_family(array):
+    return {
+        "_class": "Family",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "father_handle": array[2],
+        "mother_handle": array[3],
+        "child_ref_list": [convert_child_ref(ref) for ref in array[4]],
+        "type": convert_type("FamilyRelType", array[5]),
+        "event_ref_list": [convert_event_ref(ref) for ref in array[6]],
+        "media_list": [convert_media_ref(ref) for ref in array[7]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[8]],
+        "lds_seal_list": [convert_ord(ord) for ord in array[9]],
+        "citation_list": array[10],  # handles
+        "note_list": array[11],  # handles
+        "change": array[12],
+        "tag_list": array[13],
+        "private": array[14],
+    }
+
+
+def convert_child_ref(array):
+    return {
+        "_class": "ChildRef",
+        "private": array[0],
+        "citation_list": array[1],
+        "note_list": array[2],
+        "ref": array[3],
+        "frel": convert_type("ChildRefType", array[4]),
+        "mrel": convert_type("ChildRefType", array[5]),
+    }
+
+
+def convert_event(array):
+    return {
+        "_class": "Event",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "type": convert_type("EventType", array[2]),
+        "date": convert_date(array[3]),
+        "description": array[4],
+        "place": array[5],
+        "citation_list": array[6],
+        "note_list": array[7],
+        "media_list": [convert_media_ref(ref) for ref in array[8]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[9]],
+        "change": array[10],
+        "tag_list": array[11],
+        "private": array[12],
+    }
+
+
+def convert_place(array):
+    return {
+        "_class": "Place",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "title": array[2],
+        "long": array[3],
+        "lat": array[4],
+        "placeref_list": [convert_place_ref(ref) for ref in array[5]],
+        "name": convert_place_name(array[6]),
+        "alt_names": [convert_place_name(name) for name in array[7]],
+        "place_type": convert_type("PlaceType", array[8]),
+        "code": array[9],
+        "alt_loc": [convert_location(loc) for loc in array[10]],
+        "urls": [convert_url(url) for url in array[11]],
+        "media_list": [convert_media_ref(ref) for ref in array[12]],
+        "citation_list": array[13],
+        "note_list": array[14],
+        "change": array[15],
+        "tag_list": array[16],
+        "private": array[17],
+    }
+
+
+def convert_location(array):
+    return {
+        "_class": "Location",
+        "street": array[0][0],
+        "locality": array[0][1],
+        "city": array[0][2],
+        "county": array[0][3],
+        "state": array[0][4],
+        "country": array[0][5],
+        "postal": array[0][6],
+        "phone": array[0][7],
+        "parish": array[1],
+    }
+
+
+def convert_place_ref(array):
+    return {
+        "_class": "PlaceRef",
+        "ref": array[0],
+        "date": convert_date(array[1]),
+    }
+
+
+def convert_place_name(array):
+    return {
+        "_class": "PlaceName",
+        "value": array[0],
+        "date": convert_date(array[1]),
+        "lang": array[2],
+    }
+
+
+def convert_repository(array):
+    return {
+        "_class": "Repository",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "type": convert_type("RepositoryType", array[2]),
+        "name": array[3],
+        "note_list": array[4],
+        "address_list": [convert_address(addr) for addr in array[5]],
+        "urls": [convert_url(url) for url in array[6]],
+        "change": array[7],
+        "tag_list": array[8],
+        "private": array[9],
+    }
+
+
+def convert_source(array):
+    return {
+        "_class": "Source",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "title": array[2],
+        "author": array[3],
+        "pubinfo": array[4],
+        "note_list": array[5],
+        "media_list": [convert_media_ref(ref) for ref in array[6]],
+        "abbrev": array[7],
+        "change": array[8],
+        "attribute_list": [
+            convert_attribute("SrcAttribute", attr) for attr in array[9]
+        ],
+        "reporef_list": [convert_repo_ref(ref) for ref in array[10]],
+        "tag_list": array[11],
+        "private": array[12],
+    }
+
+
+def convert_repo_ref(array):
+    return {
+        "_class": "RepoRef",
+        "note_list": array[0],
+        "ref": array[1],
+        "call_number": array[2],
+        "media_type": convert_type("SourceMediaType", array[3]),
+        "private": array[4],
+    }
+
+
+def convert_citation(array):
+    return {
+        "_class": "Citation",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "date": convert_date(array[2]),
+        "page": array[3],
+        "confidence": array[4],
+        "source_handle": array[5],
+        "note_list": array[6],
+        "media_list": [convert_media_ref(ref) for ref in array[7]],
+        "attribute_list": [
+            convert_attribute("SrcAttribute", attr) for attr in array[8]
+        ],
+        "change": array[9],
+        "tag_list": array[10],
+        "private": array[11],
+    }
+
+
+def convert_media(array):
+    return {
+        "_class": "Media",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "path": array[2],
+        "mime": array[3],
+        "desc": array[4],
+        "checksum": array[5],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[6]],
+        "citation_list": array[7],
+        "note_list": array[8],
+        "change": array[9],
+        "date": convert_date(array[10]),
+        "tag_list": array[11],
+        "private": array[12],
+    }
+
+
+def convert_note(array):
+    return {
+        "_class": "Note",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "text": convert_styledtext(array[2]),
+        "format": array[3],
+        "type": convert_type("NoteType", array[4]),
+        "change": array[5],
+        "tag_list": array[6],
+        "private": array[7],
+    }
+
+
+def convert_styledtext(array):
+    return {
+        "_class": "StyledText",
+        "string": array[0],
+        "tags": [convert_stt(stt) for stt in array[1]],
+    }
+
+
+def convert_tag(array):
+    return {
+        "_class": "Tag",
+        "handle": array[0],
+        "name": array[1],
+        "color": array[2],
+        "priority": array[3],
+        "change": array[4],
+    }
diff --git a/gramps/gen/db/upgrade.py b/gramps/gen/db/upgrade.py
index 2c8d2738c9..6d571b220a 100644
--- a/gramps/gen/db/upgrade.py
+++ b/gramps/gen/db/upgrade.py
@@ -55,6 +55,8 @@
     TAG_KEY,
 )
 from ..const import GRAMPS_LOCALE as glocale
+from .conversion_tools import convert_21
+from gramps.gen.lib.serialize import to_dict
 
 _ = glocale.translation.gettext
 
@@ -73,36 +75,42 @@ def gramps_upgrade_21(self):
     self.set_total(length)
 
     # First, do metadata:
+    self.set_serializer("blob")
     self.upgrade_table_for_json_data("metadata")
     keys = self._get_metadata_keys()
     for key in keys:
         self.set_serializer("blob")
         value = self._get_metadata(key, "not-found")
         if value != "not-found":
+            # Save to json_data in current format
             self.set_serializer("json")
-            self._set_metadata(key, value)
+            self._set_metadata(key, convert_21("metadata", value))
 
     self._txn_begin()
     for table_name in self._get_table_func():
         # For each table, alter the database in an appropriate way:
         self.upgrade_table_for_json_data(table_name.lower())
 
-        get_obj_from_handle = self._get_table_func(table_name, "handle_func")
+        get_array_from_handle = self._get_table_func(table_name, "raw_func")
+        get_object_from_handle = self._get_table_func(table_name, "handle_func")
         get_handles = self._get_table_func(table_name, "handles_func")
         commit_func = self._get_table_func(table_name, "commit_func")
         key = CLASS_TO_KEY_MAP[table_name]
         for handle in get_handles():
-            # Initially, serializer must be set to blob:
+            # Load from blob:
             self.set_serializer("blob")
-            obj = get_obj_from_handle(handle)
-            # Force the save in json_data:
-            raw = to_dict(obj)
+            array = get_array_from_handle(handle)
+            # Save to json_data in version 21 format
+            json_data = convert_21(table_name, array)
             self.set_serializer("json")
-            self._commit_raw(raw, key)
+            # We can use commit_raw as long as json_data
+            # has "handle", "_class", and uses json.dumps()
+            self._commit_raw(json_data, key)
             self.update()
 
     self._txn_commit()
     # Bump up database version. Separate transaction to save metadata.
+    self.set_serializer("json")
     self._set_metadata("version", 21)
 
 
diff --git a/gramps/gen/lib/surname.py b/gramps/gen/lib/surname.py
index 84aff9e3e2..577f9a472e 100644
--- a/gramps/gen/lib/surname.py
+++ b/gramps/gen/lib/surname.py
@@ -69,6 +69,25 @@ def __init__(self, source=None, data=None):
         if data:
             self.unserialize(data)
 
+    def get_object_state(self):
+        """
+        Get the current object state as a dictionary.
+        """
+        attr_dict = dict(
+            (key, value)
+            for key, value in self.__dict__.items()
+            if not key.startswith("_")
+        )
+        attr_dict["_class"] = self.__class__.__name__
+        return attr_dict
+
+    def set_object_state(self, attr_dict):
+        """
+        Set the current object state using information provided in the given
+        dictionary.
+        """
+        self.__dict__.update(attr_dict)
+
     def serialize(self):
         """
         Convert the object to a serialized tuple of data.

From 05b936bd2240749d1abbf006945c230477971f56 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 14:14:24 -0500
Subject: [PATCH 38/46] Fixes after mass conversion

---
 gramps/gen/db/conversion_tools.py | 55 +++++++++----------------------
 gramps/gen/db/upgrade.py          |  2 +-
 gramps/gen/lib/serialize.py       |  6 ++--
 3 files changed, 21 insertions(+), 42 deletions(-)

diff --git a/gramps/gen/db/conversion_tools.py b/gramps/gen/db/conversion_tools.py
index 033ebf3e87..cb7b8af5db 100644
--- a/gramps/gen/db/conversion_tools.py
+++ b/gramps/gen/db/conversion_tools.py
@@ -40,24 +40,6 @@ def convert_21(classname, array):
         return convert_note(array)
     elif classname == "Tag":
         return convert_tag(array)
-    elif classname == "metadata":
-        value = array
-        type_name = type(value).__name__
-        if type_name in ("set", "tuple"):
-            value = list(value)
-        elif type_name == "Researcher":
-            value = convert_researcher(value)
-        elif type_name in ("int", "str", "list"):
-            pass
-        else:
-            raise Exception("metadata unknown type: %s" % type_name)
-
-        data = {
-            "type": type_name,
-            "value": value,
-        }
-        return data
-
     else:
         raise Exception("unknown class: %s" % classname)
 
@@ -151,28 +133,21 @@ def convert_address(array):
         "citation_list": array[1],  # handles
         "note_list": array[2],  # handles
         "date": convert_date(array[3]),
-        "location": convert_location(array[4]),
-    }
-
-
-def convert_location(array):
-    return {
-        "_class": "Location",
-        "street": array[0],
-        "locality": array[1],
-        "city": array[2],
-        "county": array[3],
-        "state": array[4],
-        "country": array[5],
-        "postal": array[6],
-        "phone": array[7],
+        "street": array[4][0],
+        "locality": array[4][1],
+        "city": array[4][2],
+        "county": array[4][3],
+        "state": array[4][4],
+        "country": array[4][5],
+        "postal": array[4][6],
+        "phone": array[4][7],
     }
 
 
 def convert_media_ref(array):
     return {
         "_class": "MediaRef",
-        "privacy": array[0],
+        "private": array[0],
         "citation_list": array[1],  # handles
         "note_list": array[2],  # handles
         "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
@@ -184,7 +159,7 @@ def convert_media_ref(array):
 def convert_event_ref(array):
     return {
         "_class": "EventRef",
-        "privacy": array[0],
+        "private": array[0],
         "citation_list": array[1],  # handles
         "note_list": array[2],  # handles
         "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
@@ -217,13 +192,13 @@ def convert_name(array):
         "_class": "Name",
         "private": array[0],
         "citation_list": array[1],
-        "note": array[2],
+        "note_list": array[2],  # handles
         "date": convert_date(array[3]),
         "first_name": array[4],
         "surname_list": [convert_surname(name) for name in array[5]],
         "suffix": array[6],
         "title": array[7],
-        "type": convert_type("Note", array[8]),
+        "type": convert_type("NameType", array[8]),
         "group_as": array[9],
         "sort_as": array[10],
         "display_as": array[11],
@@ -239,7 +214,7 @@ def convert_surname(array):
         "surname": array[0],
         "prefix": array[1],
         "primary": array[2],
-        "origin_type": convert_type("NameOriginType", array[3]),
+        "origintype": convert_type("NameOriginType", array[3]),
         "connector": array[4],
     }
 
@@ -255,6 +230,7 @@ def convert_date(array):
             "text": "",
             "sortval": 0,
             "newyear": 0,
+            "format": None,
         }
     else:
         return {
@@ -266,6 +242,7 @@ def convert_date(array):
             "text": array[4],
             "sortval": array[5],
             "newyear": array[6],
+            "format": None,
         }
 
 
@@ -298,7 +275,7 @@ def convert_family(array):
         "event_ref_list": [convert_event_ref(ref) for ref in array[6]],
         "media_list": [convert_media_ref(ref) for ref in array[7]],
         "attribute_list": [convert_attribute("Attribute", attr) for attr in array[8]],
-        "lds_seal_list": [convert_ord(ord) for ord in array[9]],
+        "lds_ord_list": [convert_ord(ord) for ord in array[9]],
         "citation_list": array[10],  # handles
         "note_list": array[11],  # handles
         "change": array[12],
diff --git a/gramps/gen/db/upgrade.py b/gramps/gen/db/upgrade.py
index 6d571b220a..383f98951c 100644
--- a/gramps/gen/db/upgrade.py
+++ b/gramps/gen/db/upgrade.py
@@ -84,7 +84,7 @@ def gramps_upgrade_21(self):
         if value != "not-found":
             # Save to json_data in current format
             self.set_serializer("json")
-            self._set_metadata(key, convert_21("metadata", value))
+            self._set_metadata(key, value)
 
     self._txn_begin()
     for table_name in self._get_table_func():
diff --git a/gramps/gen/lib/serialize.py b/gramps/gen/lib/serialize.py
index 526026e3aa..40ebe1568b 100644
--- a/gramps/gen/lib/serialize.py
+++ b/gramps/gen/lib/serialize.py
@@ -199,10 +199,12 @@ def metadata_to_object(string):
             return set(doc["value"])
         elif type_name == "tuple":
             return tuple(doc["value"])
-        elif type_name == "dict" and "_class" in doc:
+        elif type_name == "dict":
+            return doc["value"]
+        elif type_name == "Researcher":
             return from_dict(doc["value"])
         else:
-            return from_json(json.dumps(doc["value"]))
+            return doc["value"]
 
     @staticmethod
     def object_to_metadata(value):

From e699af3ea4b1660744177e6ba68e7b0679e116b5 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 14:33:41 -0500
Subject: [PATCH 39/46] Don't eat exception

---
 gramps/plugins/db/dbapi/dbapi.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index ab747c93dc..8ce7ef6ffa 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -97,11 +97,9 @@ def upgrade_table_for_json_data(self, table_name):
         A DBAPI level method for upgrading the given table
         adding a json_data column.
         """
-        try:
+        if not self.dbapi.column_exists(table_name, "json_data"):
             self.dbapi.execute("ALTER TABLE %s ADD COLUMN json_data TEXT;" % table_name)
             self.dbapi.commit()
-        except Exception:
-            pass
 
     def _schema_exists(self):
         """

From f5c7c0d021923c3718866c393896bc2e0e9f3b00 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 14:49:17 -0500
Subject: [PATCH 40/46] Put entire set of changes in one transaction

---
 gramps/gen/db/generic.py         |  4 +++-
 gramps/gen/db/upgrade.py         |  9 +++++----
 gramps/plugins/db/dbapi/dbapi.py | 11 +++++++----
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py
index d1949fbb4a..7036259e71 100644
--- a/gramps/gen/db/generic.py
+++ b/gramps/gen/db/generic.py
@@ -942,10 +942,12 @@ def _get_metadata(self, key, default=[]):
         """
         raise NotImplementedError
 
-    def _set_metadata(self, key, value):
+    def _set_metadata(self, key, value, use_txn=True):
         """
         key: string
         value: item, will be serialized here
+
+        Note: if use_txn, then begin/commit txn
         """
         raise NotImplementedError
 
diff --git a/gramps/gen/db/upgrade.py b/gramps/gen/db/upgrade.py
index 383f98951c..68450db57d 100644
--- a/gramps/gen/db/upgrade.py
+++ b/gramps/gen/db/upgrade.py
@@ -75,6 +75,8 @@ def gramps_upgrade_21(self):
     self.set_total(length)
 
     # First, do metadata:
+
+    self._txn_begin()
     self.set_serializer("blob")
     self.upgrade_table_for_json_data("metadata")
     keys = self._get_metadata_keys()
@@ -84,9 +86,8 @@ def gramps_upgrade_21(self):
         if value != "not-found":
             # Save to json_data in current format
             self.set_serializer("json")
-            self._set_metadata(key, value)
+            self._set_metadata(key, value, use_txn=False)
 
-    self._txn_begin()
     for table_name in self._get_table_func():
         # For each table, alter the database in an appropriate way:
         self.upgrade_table_for_json_data(table_name.lower())
@@ -108,10 +109,10 @@ def gramps_upgrade_21(self):
             self._commit_raw(json_data, key)
             self.update()
 
+    self.set_serializer("json")
+    self._set_metadata("version", 21, use_txn=False)
     self._txn_commit()
     # Bump up database version. Separate transaction to save metadata.
-    self.set_serializer("json")
-    self._set_metadata("version", 21)
 
 
 def gramps_upgrade_20(self):
diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py
index 8ce7ef6ffa..781657429e 100644
--- a/gramps/plugins/db/dbapi/dbapi.py
+++ b/gramps/plugins/db/dbapi/dbapi.py
@@ -99,7 +99,6 @@ def upgrade_table_for_json_data(self, table_name):
         """
         if not self.dbapi.column_exists(table_name, "json_data"):
             self.dbapi.execute("ALTER TABLE %s ADD COLUMN json_data TEXT;" % table_name)
-            self.dbapi.commit()
 
     def _schema_exists(self):
         """
@@ -390,12 +389,15 @@ def _get_metadata(self, key, default="_"):
             return []
         return default
 
-    def _set_metadata(self, key, value):
+    def _set_metadata(self, key, value, use_txn=True):
         """
         key: string
         value: item, will be serialized here
+
+        Note: if use_txn, then begin/commit txn
         """
-        self._txn_begin()
+        if use_txn:
+            self._txn_begin()
         self.dbapi.execute("SELECT 1 FROM metadata WHERE setting = ?", [key])
         row = self.dbapi.fetchone()
         if row:
@@ -408,7 +410,8 @@ def _set_metadata(self, key, value):
                 f"INSERT INTO metadata (setting, {self.serializer.metadata_field}) VALUES (?, ?)",
                 [key, self.serializer.object_to_metadata(value)],
             )
-        self._txn_commit()
+        if use_txn:
+            self._txn_commit()
 
     def get_name_group_keys(self):
         """

From c86a2c25353f8f208712b658b85ad0d0aab17896 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 15:35:50 -0500
Subject: [PATCH 41/46] Added a couple of missing defaults: family.complete,
 media.thumb

---
 gramps/gen/db/conversion_tools.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/gramps/gen/db/conversion_tools.py b/gramps/gen/db/conversion_tools.py
index cb7b8af5db..47af3f0971 100644
--- a/gramps/gen/db/conversion_tools.py
+++ b/gramps/gen/db/conversion_tools.py
@@ -281,6 +281,7 @@ def convert_family(array):
         "change": array[12],
         "tag_list": array[13],
         "private": array[14],
+        "complete": 0,
     }
 
 
@@ -455,6 +456,7 @@ def convert_media(array):
         "date": convert_date(array[10]),
         "tag_list": array[11],
         "private": array[12],
+        "thumb": None,
     }
 
 

From 99e53c3835d7f40e703b8ae9009559869e8c7953 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 17:24:57 -0500
Subject: [PATCH 42/46] A manual test script for validating conversion

---
 test/blog_to_json_test.py | 41 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 test/blog_to_json_test.py

diff --git a/test/blog_to_json_test.py b/test/blog_to_json_test.py
new file mode 100644
index 0000000000..bfdf0ead90
--- /dev/null
+++ b/test/blog_to_json_test.py
@@ -0,0 +1,41 @@
+from gramps.gen.db.utils import open_database
+from gramps.gen.lib.serialize import to_dict, from_dict
+from gramps.gen.db.conversion_tools import convert_21
+
+# 1. Prepare: create a table named "Example" in version 20
+#    containing the blob data
+
+# 2. Open the database in latest Gramps to convert the database
+#    to version 21
+
+db = open_database("Example")
+
+# This is a version 21 database
+
+# But we tell it to use the blob data:
+
+db.set_serializer("blob")
+
+for table_name in db._get_table_func():
+    print("Testing %s..." % table_name)
+    get_array_from_handle = db._get_table_func(table_name, "raw_func")
+    iter_objects = db._get_table_func(table_name, "iter_func")
+
+    for obj in iter_objects():
+        # We convert the object into the JSON dicts:
+        json_data = to_dict(obj)
+
+        # We get the blob array:
+        array = get_array_from_handle(obj.handle)
+
+        # We convert the array to JSON dict using the
+        # conversion code to convert array directly to
+        # dict
+
+        convert_data = convert_21(table_name, array)
+
+        # Now we make sure they are identical in types
+        # and values:
+        assert convert_data == json_data
+
+db.close()

From 8462bbe373a3d0032449784a460f124252706a03 Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 18:54:17 -0500
Subject: [PATCH 43/46] Rename

---
 test/{blog_to_json_test.py => blob_to_json_test.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename test/{blog_to_json_test.py => blob_to_json_test.py} (100%)

diff --git a/test/blog_to_json_test.py b/test/blob_to_json_test.py
similarity index 100%
rename from test/blog_to_json_test.py
rename to test/blob_to_json_test.py

From a124a1aa9db71fab8fbefd37f3a2d8f2ff6e8cbe Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Mon, 18 Nov 2024 18:55:39 -0500
Subject: [PATCH 44/46] table -> database

---
 test/blob_to_json_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/blob_to_json_test.py b/test/blob_to_json_test.py
index bfdf0ead90..766657bda5 100644
--- a/test/blob_to_json_test.py
+++ b/test/blob_to_json_test.py
@@ -2,7 +2,7 @@
 from gramps.gen.lib.serialize import to_dict, from_dict
 from gramps.gen.db.conversion_tools import convert_21
 
-# 1. Prepare: create a table named "Example" in version 20
+# 1. Prepare: create a database named "Example" in version 20
 #    containing the blob data
 
 # 2. Open the database in latest Gramps to convert the database

From 6832a43abca18efce871c0700c29976c2926b3fe Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Tue, 19 Nov 2024 17:11:19 -0500
Subject: [PATCH 45/46] Moved conversion tools around a bit

---
 gramps/gen/db/conversion_tools/__init__.py    |  21 +
 .../gen/db/conversion_tools/conversion_21.py  | 493 ++++++++++++++++++
 2 files changed, 514 insertions(+)
 create mode 100644 gramps/gen/db/conversion_tools/__init__.py
 create mode 100644 gramps/gen/db/conversion_tools/conversion_21.py

diff --git a/gramps/gen/db/conversion_tools/__init__.py b/gramps/gen/db/conversion_tools/__init__.py
new file mode 100644
index 0000000000..0743cd6d78
--- /dev/null
+++ b/gramps/gen/db/conversion_tools/__init__.py
@@ -0,0 +1,21 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2024 Doug Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from .conversion_21 import convert_21
diff --git a/gramps/gen/db/conversion_tools/conversion_21.py b/gramps/gen/db/conversion_tools/conversion_21.py
new file mode 100644
index 0000000000..47af3f0971
--- /dev/null
+++ b/gramps/gen/db/conversion_tools/conversion_21.py
@@ -0,0 +1,493 @@
+#
+# Gramps - a GTK+/GNOME based genealogy program
+#
+# Copyright (C) 2024 Doug Blank <doug.blank@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+
+def convert_21(classname, array):
+    if classname == "Person":
+        return convert_person(array)
+    elif classname == "Family":
+        return convert_family(array)
+    elif classname == "Event":
+        return convert_event(array)
+    elif classname == "Place":
+        return convert_place(array)
+    elif classname == "Repository":
+        return convert_repository(array)
+    elif classname == "Source":
+        return convert_source(array)
+    elif classname == "Citation":
+        return convert_citation(array)
+    elif classname == "Media":
+        return convert_media(array)
+    elif classname == "Note":
+        return convert_note(array)
+    elif classname == "Tag":
+        return convert_tag(array)
+    else:
+        raise Exception("unknown class: %s" % classname)
+
+
+def convert_researcher(researcher):
+    # This can be a problem if the Researcher class
+    # changes!
+    return {
+        "_class": "Researcher",
+        "street": researcher.street,
+        "city": researcher.city,
+        "county": researcher.county,
+        "state": researcher.state,
+        "country": researcher.country,
+        "postal": researcher.postal,
+        "phone": researcher.phone,
+        "locality": researcher.locality,
+        "name": researcher.name,
+        "addr": researcher.addr,
+        "email": researcher.email,
+    }
+
+
+def convert_person(array):
+    return {
+        "_class": "Person",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "gender": array[2],
+        "primary_name": convert_name(array[3]),
+        "alternate_names": [convert_name(name) for name in array[4]],
+        "death_ref_index": array[5],
+        "birth_ref_index": array[6],
+        "event_ref_list": [convert_event_ref(ref) for ref in array[7]],
+        "family_list": array[8],
+        "parent_family_list": array[9],
+        "media_list": [convert_media_ref(ref) for ref in array[10]],
+        "address_list": [convert_address(address) for address in array[11]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[12]],
+        "urls": [convert_url(url) for url in array[13]],
+        "lds_ord_list": [convert_ord(ord) for ord in array[14]],
+        "citation_list": array[15],  # handles
+        "note_list": array[16],  # handles
+        "change": array[17],
+        "tag_list": array[18],  # handles
+        "private": array[19],
+        "person_ref_list": [convert_person_ref(ref) for ref in array[20]],
+    }
+
+
+def convert_person_ref(array):
+    return {
+        "_class": "PersonRef",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "ref": array[3],
+        "rel": array[4],
+    }
+
+
+def convert_ord(array):
+    return {
+        "_class": "LdsOrd",
+        "citation_list": array[0],  # handles
+        "note_list": array[1],  # handles
+        "date": convert_date(array[2]),
+        "type": array[3],  # string
+        "place": array[4],
+        "famc": array[5],
+        "temple": array[6],
+        "status": array[7],
+        "private": array[8],
+    }
+
+
+def convert_url(array):
+    return {
+        "_class": "Url",
+        "private": array[0],
+        "path": array[1],
+        "desc": array[2],
+        "type": convert_type("UrlType", array[3]),
+    }
+
+
+def convert_address(array):
+    return {
+        "_class": "Address",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "date": convert_date(array[3]),
+        "street": array[4][0],
+        "locality": array[4][1],
+        "city": array[4][2],
+        "county": array[4][3],
+        "state": array[4][4],
+        "country": array[4][5],
+        "postal": array[4][6],
+        "phone": array[4][7],
+    }
+
+
+def convert_media_ref(array):
+    return {
+        "_class": "MediaRef",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
+        "ref": array[4],
+        "rect": list(array[5]) if array[5] is not None else None,
+    }
+
+
+def convert_event_ref(array):
+    return {
+        "_class": "EventRef",
+        "private": array[0],
+        "citation_list": array[1],  # handles
+        "note_list": array[2],  # handles
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
+        "ref": array[4],
+        "role": convert_type("EventRoleType", array[5]),
+    }
+
+
+def convert_attribute(classname, array):
+    if classname == "SrcAttribute":
+        return {
+            "_class": classname,
+            "private": array[0],
+            "type": convert_type("SrcAttributeType", array[1]),
+            "value": array[2],
+        }
+    else:
+        return {
+            "_class": classname,
+            "private": array[0],
+            "citation_list": array[1],  # handles
+            "note_list": array[2],  # handles
+            "type": convert_type("AttributeType", array[3]),
+            "value": array[4],
+        }
+
+
+def convert_name(array):
+    return {
+        "_class": "Name",
+        "private": array[0],
+        "citation_list": array[1],
+        "note_list": array[2],  # handles
+        "date": convert_date(array[3]),
+        "first_name": array[4],
+        "surname_list": [convert_surname(name) for name in array[5]],
+        "suffix": array[6],
+        "title": array[7],
+        "type": convert_type("NameType", array[8]),
+        "group_as": array[9],
+        "sort_as": array[10],
+        "display_as": array[11],
+        "call": array[12],
+        "nick": array[13],
+        "famnick": array[14],
+    }
+
+
+def convert_surname(array):
+    return {
+        "_class": "Surname",
+        "surname": array[0],
+        "prefix": array[1],
+        "primary": array[2],
+        "origintype": convert_type("NameOriginType", array[3]),
+        "connector": array[4],
+    }
+
+
+def convert_date(array):
+    if array is None:
+        return {
+            "_class": "Date",
+            "calendar": 0,
+            "modifier": 0,
+            "quality": 0,
+            "dateval": [0, 0, 0, False],
+            "text": "",
+            "sortval": 0,
+            "newyear": 0,
+            "format": None,
+        }
+    else:
+        return {
+            "_class": "Date",
+            "calendar": array[0],
+            "modifier": array[1],
+            "quality": array[2],
+            "dateval": list(array[3]),
+            "text": array[4],
+            "sortval": array[5],
+            "newyear": array[6],
+            "format": None,
+        }
+
+
+def convert_stt(array):
+    return {
+        "_class": "StyledTextTag",
+        "name": convert_type("StyledTextTagType", array[0]),
+        "value": array[1],
+        "ranges": [list(r) for r in array[2]],
+    }
+
+
+def convert_type(classname, array):
+    return {
+        "_class": classname,
+        "value": array[0],
+        "string": array[1],
+    }
+
+
+def convert_family(array):
+    return {
+        "_class": "Family",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "father_handle": array[2],
+        "mother_handle": array[3],
+        "child_ref_list": [convert_child_ref(ref) for ref in array[4]],
+        "type": convert_type("FamilyRelType", array[5]),
+        "event_ref_list": [convert_event_ref(ref) for ref in array[6]],
+        "media_list": [convert_media_ref(ref) for ref in array[7]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[8]],
+        "lds_ord_list": [convert_ord(ord) for ord in array[9]],
+        "citation_list": array[10],  # handles
+        "note_list": array[11],  # handles
+        "change": array[12],
+        "tag_list": array[13],
+        "private": array[14],
+        "complete": 0,
+    }
+
+
+def convert_child_ref(array):
+    return {
+        "_class": "ChildRef",
+        "private": array[0],
+        "citation_list": array[1],
+        "note_list": array[2],
+        "ref": array[3],
+        "frel": convert_type("ChildRefType", array[4]),
+        "mrel": convert_type("ChildRefType", array[5]),
+    }
+
+
+def convert_event(array):
+    return {
+        "_class": "Event",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "type": convert_type("EventType", array[2]),
+        "date": convert_date(array[3]),
+        "description": array[4],
+        "place": array[5],
+        "citation_list": array[6],
+        "note_list": array[7],
+        "media_list": [convert_media_ref(ref) for ref in array[8]],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[9]],
+        "change": array[10],
+        "tag_list": array[11],
+        "private": array[12],
+    }
+
+
+def convert_place(array):
+    return {
+        "_class": "Place",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "title": array[2],
+        "long": array[3],
+        "lat": array[4],
+        "placeref_list": [convert_place_ref(ref) for ref in array[5]],
+        "name": convert_place_name(array[6]),
+        "alt_names": [convert_place_name(name) for name in array[7]],
+        "place_type": convert_type("PlaceType", array[8]),
+        "code": array[9],
+        "alt_loc": [convert_location(loc) for loc in array[10]],
+        "urls": [convert_url(url) for url in array[11]],
+        "media_list": [convert_media_ref(ref) for ref in array[12]],
+        "citation_list": array[13],
+        "note_list": array[14],
+        "change": array[15],
+        "tag_list": array[16],
+        "private": array[17],
+    }
+
+
+def convert_location(array):
+    return {
+        "_class": "Location",
+        "street": array[0][0],
+        "locality": array[0][1],
+        "city": array[0][2],
+        "county": array[0][3],
+        "state": array[0][4],
+        "country": array[0][5],
+        "postal": array[0][6],
+        "phone": array[0][7],
+        "parish": array[1],
+    }
+
+
+def convert_place_ref(array):
+    return {
+        "_class": "PlaceRef",
+        "ref": array[0],
+        "date": convert_date(array[1]),
+    }
+
+
+def convert_place_name(array):
+    return {
+        "_class": "PlaceName",
+        "value": array[0],
+        "date": convert_date(array[1]),
+        "lang": array[2],
+    }
+
+
+def convert_repository(array):
+    return {
+        "_class": "Repository",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "type": convert_type("RepositoryType", array[2]),
+        "name": array[3],
+        "note_list": array[4],
+        "address_list": [convert_address(addr) for addr in array[5]],
+        "urls": [convert_url(url) for url in array[6]],
+        "change": array[7],
+        "tag_list": array[8],
+        "private": array[9],
+    }
+
+
+def convert_source(array):
+    return {
+        "_class": "Source",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "title": array[2],
+        "author": array[3],
+        "pubinfo": array[4],
+        "note_list": array[5],
+        "media_list": [convert_media_ref(ref) for ref in array[6]],
+        "abbrev": array[7],
+        "change": array[8],
+        "attribute_list": [
+            convert_attribute("SrcAttribute", attr) for attr in array[9]
+        ],
+        "reporef_list": [convert_repo_ref(ref) for ref in array[10]],
+        "tag_list": array[11],
+        "private": array[12],
+    }
+
+
+def convert_repo_ref(array):
+    return {
+        "_class": "RepoRef",
+        "note_list": array[0],
+        "ref": array[1],
+        "call_number": array[2],
+        "media_type": convert_type("SourceMediaType", array[3]),
+        "private": array[4],
+    }
+
+
+def convert_citation(array):
+    return {
+        "_class": "Citation",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "date": convert_date(array[2]),
+        "page": array[3],
+        "confidence": array[4],
+        "source_handle": array[5],
+        "note_list": array[6],
+        "media_list": [convert_media_ref(ref) for ref in array[7]],
+        "attribute_list": [
+            convert_attribute("SrcAttribute", attr) for attr in array[8]
+        ],
+        "change": array[9],
+        "tag_list": array[10],
+        "private": array[11],
+    }
+
+
+def convert_media(array):
+    return {
+        "_class": "Media",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "path": array[2],
+        "mime": array[3],
+        "desc": array[4],
+        "checksum": array[5],
+        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[6]],
+        "citation_list": array[7],
+        "note_list": array[8],
+        "change": array[9],
+        "date": convert_date(array[10]),
+        "tag_list": array[11],
+        "private": array[12],
+        "thumb": None,
+    }
+
+
+def convert_note(array):
+    return {
+        "_class": "Note",
+        "handle": array[0],
+        "gramps_id": array[1],
+        "text": convert_styledtext(array[2]),
+        "format": array[3],
+        "type": convert_type("NoteType", array[4]),
+        "change": array[5],
+        "tag_list": array[6],
+        "private": array[7],
+    }
+
+
+def convert_styledtext(array):
+    return {
+        "_class": "StyledText",
+        "string": array[0],
+        "tags": [convert_stt(stt) for stt in array[1]],
+    }
+
+
+def convert_tag(array):
+    return {
+        "_class": "Tag",
+        "handle": array[0],
+        "name": array[1],
+        "color": array[2],
+        "priority": array[3],
+        "change": array[4],
+    }

From 031c7c41f6e2d453911951a04465da82a04b17aa Mon Sep 17 00:00:00 2001
From: Doug Blank <doug.blank@gmail.com>
Date: Tue, 19 Nov 2024 17:14:10 -0500
Subject: [PATCH 46/46] Moved conversion tools around a bit

---
 gramps/gen/db/conversion_tools.py | 493 ------------------------------
 1 file changed, 493 deletions(-)
 delete mode 100644 gramps/gen/db/conversion_tools.py

diff --git a/gramps/gen/db/conversion_tools.py b/gramps/gen/db/conversion_tools.py
deleted file mode 100644
index 47af3f0971..0000000000
--- a/gramps/gen/db/conversion_tools.py
+++ /dev/null
@@ -1,493 +0,0 @@
-#
-# Gramps - a GTK+/GNOME based genealogy program
-#
-# Copyright (C) 2024 Doug Blank <doug.blank@gmail.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-
-
-def convert_21(classname, array):
-    if classname == "Person":
-        return convert_person(array)
-    elif classname == "Family":
-        return convert_family(array)
-    elif classname == "Event":
-        return convert_event(array)
-    elif classname == "Place":
-        return convert_place(array)
-    elif classname == "Repository":
-        return convert_repository(array)
-    elif classname == "Source":
-        return convert_source(array)
-    elif classname == "Citation":
-        return convert_citation(array)
-    elif classname == "Media":
-        return convert_media(array)
-    elif classname == "Note":
-        return convert_note(array)
-    elif classname == "Tag":
-        return convert_tag(array)
-    else:
-        raise Exception("unknown class: %s" % classname)
-
-
-def convert_researcher(researcher):
-    # This can be a problem if the Researcher class
-    # changes!
-    return {
-        "_class": "Researcher",
-        "street": researcher.street,
-        "city": researcher.city,
-        "county": researcher.county,
-        "state": researcher.state,
-        "country": researcher.country,
-        "postal": researcher.postal,
-        "phone": researcher.phone,
-        "locality": researcher.locality,
-        "name": researcher.name,
-        "addr": researcher.addr,
-        "email": researcher.email,
-    }
-
-
-def convert_person(array):
-    return {
-        "_class": "Person",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "gender": array[2],
-        "primary_name": convert_name(array[3]),
-        "alternate_names": [convert_name(name) for name in array[4]],
-        "death_ref_index": array[5],
-        "birth_ref_index": array[6],
-        "event_ref_list": [convert_event_ref(ref) for ref in array[7]],
-        "family_list": array[8],
-        "parent_family_list": array[9],
-        "media_list": [convert_media_ref(ref) for ref in array[10]],
-        "address_list": [convert_address(address) for address in array[11]],
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[12]],
-        "urls": [convert_url(url) for url in array[13]],
-        "lds_ord_list": [convert_ord(ord) for ord in array[14]],
-        "citation_list": array[15],  # handles
-        "note_list": array[16],  # handles
-        "change": array[17],
-        "tag_list": array[18],  # handles
-        "private": array[19],
-        "person_ref_list": [convert_person_ref(ref) for ref in array[20]],
-    }
-
-
-def convert_person_ref(array):
-    return {
-        "_class": "PersonRef",
-        "private": array[0],
-        "citation_list": array[1],  # handles
-        "note_list": array[2],  # handles
-        "ref": array[3],
-        "rel": array[4],
-    }
-
-
-def convert_ord(array):
-    return {
-        "_class": "LdsOrd",
-        "citation_list": array[0],  # handles
-        "note_list": array[1],  # handles
-        "date": convert_date(array[2]),
-        "type": array[3],  # string
-        "place": array[4],
-        "famc": array[5],
-        "temple": array[6],
-        "status": array[7],
-        "private": array[8],
-    }
-
-
-def convert_url(array):
-    return {
-        "_class": "Url",
-        "private": array[0],
-        "path": array[1],
-        "desc": array[2],
-        "type": convert_type("UrlType", array[3]),
-    }
-
-
-def convert_address(array):
-    return {
-        "_class": "Address",
-        "private": array[0],
-        "citation_list": array[1],  # handles
-        "note_list": array[2],  # handles
-        "date": convert_date(array[3]),
-        "street": array[4][0],
-        "locality": array[4][1],
-        "city": array[4][2],
-        "county": array[4][3],
-        "state": array[4][4],
-        "country": array[4][5],
-        "postal": array[4][6],
-        "phone": array[4][7],
-    }
-
-
-def convert_media_ref(array):
-    return {
-        "_class": "MediaRef",
-        "private": array[0],
-        "citation_list": array[1],  # handles
-        "note_list": array[2],  # handles
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
-        "ref": array[4],
-        "rect": list(array[5]) if array[5] is not None else None,
-    }
-
-
-def convert_event_ref(array):
-    return {
-        "_class": "EventRef",
-        "private": array[0],
-        "citation_list": array[1],  # handles
-        "note_list": array[2],  # handles
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]],
-        "ref": array[4],
-        "role": convert_type("EventRoleType", array[5]),
-    }
-
-
-def convert_attribute(classname, array):
-    if classname == "SrcAttribute":
-        return {
-            "_class": classname,
-            "private": array[0],
-            "type": convert_type("SrcAttributeType", array[1]),
-            "value": array[2],
-        }
-    else:
-        return {
-            "_class": classname,
-            "private": array[0],
-            "citation_list": array[1],  # handles
-            "note_list": array[2],  # handles
-            "type": convert_type("AttributeType", array[3]),
-            "value": array[4],
-        }
-
-
-def convert_name(array):
-    return {
-        "_class": "Name",
-        "private": array[0],
-        "citation_list": array[1],
-        "note_list": array[2],  # handles
-        "date": convert_date(array[3]),
-        "first_name": array[4],
-        "surname_list": [convert_surname(name) for name in array[5]],
-        "suffix": array[6],
-        "title": array[7],
-        "type": convert_type("NameType", array[8]),
-        "group_as": array[9],
-        "sort_as": array[10],
-        "display_as": array[11],
-        "call": array[12],
-        "nick": array[13],
-        "famnick": array[14],
-    }
-
-
-def convert_surname(array):
-    return {
-        "_class": "Surname",
-        "surname": array[0],
-        "prefix": array[1],
-        "primary": array[2],
-        "origintype": convert_type("NameOriginType", array[3]),
-        "connector": array[4],
-    }
-
-
-def convert_date(array):
-    if array is None:
-        return {
-            "_class": "Date",
-            "calendar": 0,
-            "modifier": 0,
-            "quality": 0,
-            "dateval": [0, 0, 0, False],
-            "text": "",
-            "sortval": 0,
-            "newyear": 0,
-            "format": None,
-        }
-    else:
-        return {
-            "_class": "Date",
-            "calendar": array[0],
-            "modifier": array[1],
-            "quality": array[2],
-            "dateval": list(array[3]),
-            "text": array[4],
-            "sortval": array[5],
-            "newyear": array[6],
-            "format": None,
-        }
-
-
-def convert_stt(array):
-    return {
-        "_class": "StyledTextTag",
-        "name": convert_type("StyledTextTagType", array[0]),
-        "value": array[1],
-        "ranges": [list(r) for r in array[2]],
-    }
-
-
-def convert_type(classname, array):
-    return {
-        "_class": classname,
-        "value": array[0],
-        "string": array[1],
-    }
-
-
-def convert_family(array):
-    return {
-        "_class": "Family",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "father_handle": array[2],
-        "mother_handle": array[3],
-        "child_ref_list": [convert_child_ref(ref) for ref in array[4]],
-        "type": convert_type("FamilyRelType", array[5]),
-        "event_ref_list": [convert_event_ref(ref) for ref in array[6]],
-        "media_list": [convert_media_ref(ref) for ref in array[7]],
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[8]],
-        "lds_ord_list": [convert_ord(ord) for ord in array[9]],
-        "citation_list": array[10],  # handles
-        "note_list": array[11],  # handles
-        "change": array[12],
-        "tag_list": array[13],
-        "private": array[14],
-        "complete": 0,
-    }
-
-
-def convert_child_ref(array):
-    return {
-        "_class": "ChildRef",
-        "private": array[0],
-        "citation_list": array[1],
-        "note_list": array[2],
-        "ref": array[3],
-        "frel": convert_type("ChildRefType", array[4]),
-        "mrel": convert_type("ChildRefType", array[5]),
-    }
-
-
-def convert_event(array):
-    return {
-        "_class": "Event",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "type": convert_type("EventType", array[2]),
-        "date": convert_date(array[3]),
-        "description": array[4],
-        "place": array[5],
-        "citation_list": array[6],
-        "note_list": array[7],
-        "media_list": [convert_media_ref(ref) for ref in array[8]],
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[9]],
-        "change": array[10],
-        "tag_list": array[11],
-        "private": array[12],
-    }
-
-
-def convert_place(array):
-    return {
-        "_class": "Place",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "title": array[2],
-        "long": array[3],
-        "lat": array[4],
-        "placeref_list": [convert_place_ref(ref) for ref in array[5]],
-        "name": convert_place_name(array[6]),
-        "alt_names": [convert_place_name(name) for name in array[7]],
-        "place_type": convert_type("PlaceType", array[8]),
-        "code": array[9],
-        "alt_loc": [convert_location(loc) for loc in array[10]],
-        "urls": [convert_url(url) for url in array[11]],
-        "media_list": [convert_media_ref(ref) for ref in array[12]],
-        "citation_list": array[13],
-        "note_list": array[14],
-        "change": array[15],
-        "tag_list": array[16],
-        "private": array[17],
-    }
-
-
-def convert_location(array):
-    return {
-        "_class": "Location",
-        "street": array[0][0],
-        "locality": array[0][1],
-        "city": array[0][2],
-        "county": array[0][3],
-        "state": array[0][4],
-        "country": array[0][5],
-        "postal": array[0][6],
-        "phone": array[0][7],
-        "parish": array[1],
-    }
-
-
-def convert_place_ref(array):
-    return {
-        "_class": "PlaceRef",
-        "ref": array[0],
-        "date": convert_date(array[1]),
-    }
-
-
-def convert_place_name(array):
-    return {
-        "_class": "PlaceName",
-        "value": array[0],
-        "date": convert_date(array[1]),
-        "lang": array[2],
-    }
-
-
-def convert_repository(array):
-    return {
-        "_class": "Repository",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "type": convert_type("RepositoryType", array[2]),
-        "name": array[3],
-        "note_list": array[4],
-        "address_list": [convert_address(addr) for addr in array[5]],
-        "urls": [convert_url(url) for url in array[6]],
-        "change": array[7],
-        "tag_list": array[8],
-        "private": array[9],
-    }
-
-
-def convert_source(array):
-    return {
-        "_class": "Source",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "title": array[2],
-        "author": array[3],
-        "pubinfo": array[4],
-        "note_list": array[5],
-        "media_list": [convert_media_ref(ref) for ref in array[6]],
-        "abbrev": array[7],
-        "change": array[8],
-        "attribute_list": [
-            convert_attribute("SrcAttribute", attr) for attr in array[9]
-        ],
-        "reporef_list": [convert_repo_ref(ref) for ref in array[10]],
-        "tag_list": array[11],
-        "private": array[12],
-    }
-
-
-def convert_repo_ref(array):
-    return {
-        "_class": "RepoRef",
-        "note_list": array[0],
-        "ref": array[1],
-        "call_number": array[2],
-        "media_type": convert_type("SourceMediaType", array[3]),
-        "private": array[4],
-    }
-
-
-def convert_citation(array):
-    return {
-        "_class": "Citation",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "date": convert_date(array[2]),
-        "page": array[3],
-        "confidence": array[4],
-        "source_handle": array[5],
-        "note_list": array[6],
-        "media_list": [convert_media_ref(ref) for ref in array[7]],
-        "attribute_list": [
-            convert_attribute("SrcAttribute", attr) for attr in array[8]
-        ],
-        "change": array[9],
-        "tag_list": array[10],
-        "private": array[11],
-    }
-
-
-def convert_media(array):
-    return {
-        "_class": "Media",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "path": array[2],
-        "mime": array[3],
-        "desc": array[4],
-        "checksum": array[5],
-        "attribute_list": [convert_attribute("Attribute", attr) for attr in array[6]],
-        "citation_list": array[7],
-        "note_list": array[8],
-        "change": array[9],
-        "date": convert_date(array[10]),
-        "tag_list": array[11],
-        "private": array[12],
-        "thumb": None,
-    }
-
-
-def convert_note(array):
-    return {
-        "_class": "Note",
-        "handle": array[0],
-        "gramps_id": array[1],
-        "text": convert_styledtext(array[2]),
-        "format": array[3],
-        "type": convert_type("NoteType", array[4]),
-        "change": array[5],
-        "tag_list": array[6],
-        "private": array[7],
-    }
-
-
-def convert_styledtext(array):
-    return {
-        "_class": "StyledText",
-        "string": array[0],
-        "tags": [convert_stt(stt) for stt in array[1]],
-    }
-
-
-def convert_tag(array):
-    return {
-        "_class": "Tag",
-        "handle": array[0],
-        "name": array[1],
-        "color": array[2],
-        "priority": array[3],
-        "change": array[4],
-    }