feat: remove missing packages from index during verification

ckanext/federated_index/logic/action.py

@@ -10,6 +10,7 @@
 import ckan.plugins.toolkit as tk
 from ckan import model
 from ckan.lib import search
+from ckan.lib.search.query import solr_literal
 from ckan.logic import validate
 
 from ckanext.federated_index import config, interfaces, shared, storage
@@ -56,6 +57,17 @@ def federated_index_profile_refresh(
 
     if data_dict["verify"]:
         for pkg_id in profile.check_ids(p["id"] for p in db.scan()):
+            mangled = db.get(pkg_id)
+            if not mangled:
+                continue
+
+            for plugin in p.PluginImplementations(interfaces.IFederatedIndex):
+                mangled = plugin.federated_index_mangle_package(mangled, profile)  # noqa: PLW2901
+                tk.get_action("federated_index_profile_clear")(
+                    tk.fresh_context(context),
+                    {"profile": profile, "ids": [mangled["id"]]},
+                )
+
             db.remove(pkg_id)
 
     for pkg in profile.fetch_packages(payload):
@@ -119,7 +131,6 @@ def federated_index_profile_index(
     for pkg_dict in packages:
         for plugin in p.PluginImplementations(interfaces.IFederatedIndex):
             pkg_dict = plugin.federated_index_mangle_package(pkg_dict, profile)  # noqa: PLW2901
-
         if model.Package.get(pkg_dict["name"]):
             log.warning("Package with name %s already exists", pkg_dict["name"])
             continue
@@ -184,6 +195,9 @@ def federated_index_profile_clear(
     conn = search.make_connection()
     query = f"+{config.profile_field()}:{profile.id}"
 
+    if ids := data_dict.get("ids"):
+        query += " id:({})".format(" OR ".join(map(solr_literal, ids)))
+
     resp = conn.search(q=query, rows=0)
 
     conn.delete(q=query)

ckanext/federated_index/logic/schema.py

@@ -54,9 +54,12 @@ def profile_index(
 def profile_clear(
     not_empty: types.Validator,
     federated_index_profile: types.Validator,
+    json_list_or_string: types.Validator,
+    ignore_missing: types.Validator,
 ) -> types.Schema:
     return {
         "profile": [not_empty, federated_index_profile],
+        "ids": [ignore_missing, json_list_or_string],
     }
 
 

ckanext/federated_index/tests/conftest.py

@@ -26,7 +26,7 @@ def call_action(  # noqa: PLR0913
         return super().call_action(action, data_dict, context, apikey, files)
 
 
-class TestFederatedIndexPlugin(p.Plugin):
+class TestFederatedIndexPlugin(p.SingletonPlugin):
     p.implements(interfaces.IFederatedIndex, inherit=True)
 
     def federated_index_mangle_package(
@@ -50,6 +50,9 @@ class TestProfile(shared.Profile):
     id: str = "test"
     url: str = "http://test.ckan.net"
     test_app: Any = None
+    extras: dict[str, Any] = dataclasses.field(
+        default_factory=lambda: {"search_payload": {"q": "-id:test-*"}}
+    )
 
     def get_client(self):
         return TestAppCKAN(self.test_app)

ckanext/federated_index/tests/logic/test_action.py

@@ -13,18 +13,27 @@
 class TestRefresh:
     def test_verification(self, profile: shared.Profile, package_factory: Any):
         """Removed packages dropped from storage."""
-
         pkgs = package_factory.create_batch(3)
 
-        result = call_action("federated_index_profile_refresh", profile=profile)
+        result = call_action(
+            "federated_index_profile_refresh", profile=profile, index=True
+        )
         assert result == {"count": 3, "profile": profile.id}
+        available = call_action("package_search", rows=0)["count"]
+        assert available == 6
 
         call_action("package_delete", id=pkgs[-1]["id"])
 
         result = call_action(
-            "federated_index_profile_refresh", profile=profile, verify=False
+            "federated_index_profile_refresh", profile=profile, verify=False, index=True
         )
         assert result["count"] == 3
+        available = call_action("package_search", rows=0)["count"]
+        assert available == 5
 
-        result = call_action("federated_index_profile_refresh", profile=profile)
+        result = call_action(
+            "federated_index_profile_refresh", profile=profile, index=True
+        )
         assert result["count"] == 2
+        available = call_action("package_search", rows=0)["count"]
+        assert available == 4

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = ckanext-federated-index
-version = 0.1.0
+version = 0.1.1
 description =
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -27,9 +27,7 @@ include_package_data = True
 [options.entry_points]
 ckan.plugins =
              federated_index = ckanext.federated_index.plugin:FederatedIndexPlugin
-
-ckan.test_plugins =
-                  test_federated_index = ckanext.federated_index.tests.conftest:TestFederatedIndexPlugin
+             test_federated_index = ckanext.federated_index.tests.conftest:TestFederatedIndexPlugin
 
 babel.extractors =
                  ckan = ckan.lib.extract:extract_ckan