Merge pull request #342 from singnet/senna-db-216-1

[das-atom-db#216] Removed cursor from api of get_matched*() and get_incoming_links()

CHANGELOG

@@ -1 +1,2 @@
 [#338] Change get_links() API to use link filters instead of optional parameters
+[das-atom-db#216] Removed cursor from api of get_matched*() and get_incoming_links()

hyperon_das/das.py

@@ -386,20 +386,14 @@ def get_link_handles(self, link_filter: LinkFilter) -> List[str]:
         """
         return self.query_engine.get_link_handles(link_filter)
 
-    def get_incoming_links(
-        self, atom_handle: str, **kwargs
-    ) -> tuple[int | None, IncomingLinksT | Iterator]:
+    def get_incoming_links(self, atom_handle: str, **kwargs) -> IncomingLinksT:
         """
         Retrieve all links which has the passed handle as one of its targets.
 
         Args:
             atom_handle (str): Atom's handle
 
         Keyword Args:
-            no_iterator (bool, optional): Set False to return an iterator otherwise it will return a list of Dict[str, Any].
-                If the query_engine is set to 'remote' it always return an iterator.
-                Defaults to True.
-            cursor (int, optional): Cursor position in the iterator, starts retrieving links from redis at the cursor position. Defaults to 0.
             handles_only (bool, optional): Returns a list of links handles.
 
         Returns:

hyperon_das/query_engines/local_query_engine.py

@@ -6,7 +6,7 @@
 from hyperon_das_atomdb.adapters import InMemoryDB
 from hyperon_das_atomdb.database import (
     AtomT,
-    HandlesListT,
+    HandleListT,
     IncomingLinksT,
     LinkT,
     MatchedLinksResultT,
@@ -20,13 +20,12 @@
     CustomQuery,
     LazyQueryEvaluator,
     ListIterator,
-    LocalIncomingLinks,
     QueryAnswerIterator,
 )
 from hyperon_das.client import FunctionsClient
 from hyperon_das.context import Context
 from hyperon_das.exceptions import UnexpectedQueryFormat
-from hyperon_das.link_filters import LinkFilter
+from hyperon_das.link_filters import LinkFilter, LinkFilterType
 from hyperon_das.logger import logger
 from hyperon_das.query_engines.query_engine_protocol import QueryEngine
 from hyperon_das.type_alias import Query
@@ -90,7 +89,7 @@ def _recursive_query(
                 )
             )
 
-    def _to_link_dict_list(self, db_answer: HandlesListT | MatchedTargetsListT) -> List[Dict]:
+    def _to_link_dict_list(self, db_answer: HandleListT | MatchedTargetsListT) -> List[Dict]:
         if not db_answer:
             return []
         flat_handle = isinstance(db_answer[0], str)
@@ -213,32 +212,31 @@ def get_atoms(self, handles: str, **kwargs) -> List[Dict[str, Any]]:
         return [self.local_backend.get_atom(handle, **kwargs) for handle in handles]
 
     def get_link_handles(self, link_filter: LinkFilter) -> List[str]:
-        _, answer = self._get_related_links(
-            link_type=link_filter.link_type,
-            target_types=link_filter.target_types,
-            link_targets=link_filter.targets,
-            toplevel_only=link_filter.toplevel_only,
-        )
-        return answer
+        if link_filter.filter_type == LinkFilterType.FLAT_TYPE_TEMPLATE:
+            return self.local_backend.get_matched_type_template(
+                [link_filter.link_type, *link_filter.target_types],
+                toplevel_only=link_filter.toplevel_only,
+            )
+        elif link_filter.filter_type == LinkFilterType.TARGETS:
+            return self.local_backend.get_matched_links(
+                link_filter.link_type, link_filter.targets, toplevel_only=link_filter.toplevel_only
+            )
+        elif link_filter.filter_type == LinkFilterType.NAMED_TYPE:
+            _, answer = self.local_backend.get_all_links(
+                link_filter.link_type, toplevel_only=link_filter.toplevel_only
+            )
+            return answer
+        else:
+            das_error(ValueError("Invalid LinkFilterType: {link_filter.filter_type}"))
 
     def get_links(self, link_filter: LinkFilter) -> List[LinkT]:
         handles = self.get_link_handles(link_filter)
         if not handles:
             return []
         return self._to_link_dict_list(handles)
 
-    def get_incoming_links(
-        self, atom_handle: str, **kwargs
-    ) -> tuple[int | None, IncomingLinksT | Iterator]:
-        if kwargs.get('no_iterator', True):
-            return self.local_backend.get_incoming_links(atom_handle, **kwargs)
-
-        kwargs['handles_only'] = True
-        cursor, links = self.local_backend.get_incoming_links(atom_handle, **kwargs)
-        kwargs['cursor'] = cursor
-        kwargs['backend'] = self.local_backend
-        kwargs['atom_handle'] = atom_handle
-        return cursor, LocalIncomingLinks(ListIterator(links), **kwargs)
+    def get_incoming_links(self, atom_handle: str, **kwargs) -> IncomingLinksT:
+        return self.local_backend.get_incoming_links(atom_handle, **kwargs)
 
     def query(
         self,

hyperon_das/query_engines/query_engine_protocol.py

@@ -27,7 +27,7 @@ def get_atom(self, handle: str) -> Dict[str, Any]:
         ...
 
     @abstractmethod
-    def get_atoms(self, handles: List[str]) -> List[Dict[str, Any]]:
+    def get_atoms(self, handles: List[str], **kwargs) -> List[Dict[str, Any]]:
         """
         Retrieves atoms from the database using their unique handles.
 
@@ -71,11 +71,9 @@ def get_link_handles(link_filter: LinkFilter) -> List[LinkT]:
         ...
 
     @abstractmethod
-    def get_incoming_links(
-        self, atom_handle: str, **kwargs
-    ) -> tuple[int | None, IncomingLinksT | Iterator]:
+    def get_incoming_links(self, atom_handle: str, **kwargs) -> IncomingLinksT:
         """
-        Retrieves incoming links for a specified atom handle, with optional filtering parameters.
+        Retrieves incoming links for a specified atom handle.
 
         This method fetches all links pointing to the specified atom, identified by its handle. It
         can return a simple list of links, a list of dictionaries with link details, or a list of
@@ -91,7 +89,7 @@ def get_incoming_links(
                       returned data. The exact options available depend on the implementation.
 
         Returns:
-            List[Union[dict, str, Tuple[dict, List[dict]]]]: A list containing the incoming links. The
+            List[dict]: A list containing the incoming links. The
             format of the list's elements can vary based on the provided keyword arguments and the
             implementation details.
         """

hyperon_das/query_engines/remote_query_engine.py

@@ -5,7 +5,7 @@
 from hyperon_das_atomdb.exceptions import AtomDoesNotExist
 
 from hyperon_das.cache.cache_controller import CacheController
-from hyperon_das.cache.iterators import CustomQuery, ListIterator, RemoteIncomingLinks
+from hyperon_das.cache.iterators import CustomQuery, ListIterator
 from hyperon_das.client import FunctionsClient
 from hyperon_das.context import Context
 from hyperon_das.exceptions import InvalidDASParameters, QueryParametersException
@@ -58,7 +58,7 @@ def get_atom(self, handle: str, **kwargs) -> Dict[str, Any]:
                     das_error(exception)
         return atom
 
-    def get_atoms(self, handles: List[str]) -> List[Dict[str, Any]]:
+    def get_atoms(self, handles: List[str], **kwargs) -> List[Dict[str, Any]]:
         return self.cache_controller.get_atoms(handles)
 
     def get_links(self, link_filter: LinkFilter) -> List[LinkT]:
@@ -81,18 +81,9 @@ def get_link_handles(self, link_filter: LinkFilter) -> List[str]:
         # TODO Implement get_link_handles() in faas client
         return [link['handle'] for link in self.get_links(link_filter)]
 
-    def get_incoming_links(
-        self, atom_handle: str, **kwargs
-    ) -> tuple[int | None, IncomingLinksT | Iterator]:
-        kwargs.pop('no_iterator', None)
-        if kwargs.get('cursor') is None:
-            kwargs['cursor'] = 0
-        kwargs['handles_only'] = False
-        _, links = self.local_query_engine.get_incoming_links(atom_handle, **kwargs)
-        cursor, remote_links = self.remote_das.get_incoming_links(atom_handle, **kwargs)
-        kwargs['cursor'] = cursor
-        kwargs['backend'] = self.remote_das
-        kwargs['atom_handle'] = atom_handle
+    def get_incoming_links(self, atom_handle: str, **kwargs) -> IncomingLinksT:
+        links = self.local_query_engine.get_incoming_links(atom_handle, **kwargs)
+        remote_links = self.remote_das.get_incoming_links(atom_handle, **kwargs)
         links.extend(remote_links)
         if (
             links
@@ -102,7 +93,7 @@ def get_incoming_links(
             and 'targets_document' in links[0]
         ):
             links = [(link, link.pop('targets_document', [])) for link in links]
-        return cursor, RemoteIncomingLinks(ListIterator(links), **kwargs)
+        return links
 
     def custom_query(self, index_id: str, query: Query, **kwargs) -> Iterator:
         kwargs.pop('no_iterator', None)

pyproject.toml

@@ -16,7 +16,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10"
-hyperon-das-atomdb = "0.8.2"
+hyperon-das-atomdb = "0.8.3"
 requests = "^2.31.0"
 grpcio = "^1.62.2"
 google = "^3.0.0"

tests/integration/helpers.py

@@ -88,7 +88,7 @@ def load_metta_animals_base(das: DistributedAtomSpace) -> None:
     das.add_node({"type": "Symbol", "name": 'Inheritance', "is_literal": False})
     das.add_node({"type": "Symbol", "name": 'Symbol', "is_literal": False})
     das.add_node({"type": "Symbol", "name": 'Type', "is_literal": False})
-    # das.add_node({"type": "Symbol", "name": 'MettaType', "is_literal": False})
+    das.add_node({"type": "Symbol", "name": 'MettaType', "is_literal": False})
     das.add_node({"type": "Symbol", "name": 'Expression', "is_literal": False})
     das.add_node({"type": "Symbol", "name": 'Concept', "is_literal": False})
     das.add_node({"type": "Symbol", "name": '"human"', "is_literal": True})
@@ -105,7 +105,6 @@ def load_metta_animals_base(das: DistributedAtomSpace) -> None:
     das.add_node({"type": "Symbol", "name": '"ent"', "is_literal": True})
     das.add_node({"type": "Symbol", "name": '"animal"', "is_literal": True})
     das.add_node({"type": "Symbol", "name": '"plant"', "is_literal": True})
-    das.add_node({"type": "Symbol", "name": '"plant"', "is_literal": True})
     das.add_node({"type": "Symbol", "name": ':', "is_literal": False})
     das.add_node({"type": "Symbol", "name": '<:', "is_literal": False})
 
@@ -197,6 +196,11 @@ class MettaAnimalBaseHandlesCollection:
     typedef_mark = AtomDB.node_handle('Symbol', ':')
     typedef2_mark = AtomDB.node_handle('Symbol', '<:')
 
+    node_handles = [
+        Concept, Similarity, Inheritance, Symbol, Type, Expression, human, monkey, chimp,
+        mammal, ent, animal, reptile, dinosaur, triceratops, rhino, earthworm, snake, vine,
+        plant, typedef_mark, typedef2_mark]
+
     human_typedef = AtomDB.link_handle('Expression', [typedef_mark, human, Concept])
     monkey_typedef = AtomDB.link_handle('Expression', [typedef_mark, monkey, Concept])
     chimp_typedef = AtomDB.link_handle('Expression', [typedef_mark, chimp, Concept])
@@ -241,6 +245,20 @@ class MettaAnimalBaseHandlesCollection:
     similarity_triceratops_rhino = AtomDB.link_handle('Expression', [Similarity, triceratops, rhino])
     similarity_vine_snake = AtomDB.link_handle('Expression', [Similarity, vine, snake])
     similarity_ent_human = AtomDB.link_handle('Expression', [Similarity, ent, human])
+
+    link_handles = [
+        human_typedef, monkey_typedef, chimp_typedef, snake_typedef, earthworm_typedef,
+        rhino_typedef, triceratops_typedef, vine_typedef, ent_typedef, mammal_typedef,
+        animal_typedef, reptile_typedef, dinosaur_typedef, plant_typedef, similarity_typedef,
+        inheritance_typedef, concept_typedef, similarity_human_monkey, similarity_human_chimp,
+        similarity_chimp_monkey, similarity_snake_earthworm, similarity_rhino_triceratops,
+        similarity_snake_vine, similarity_human_ent, inheritance_human_mammal,
+        inheritance_monkey_mammal, inheritance_chimp_mammal, inheritance_mammal_animal,
+        inheritance_reptile_animal, inheritance_snake_reptile, inheritance_dinosaur_reptile,
+        inheritance_triceratops_dinosaur, inheritance_earthworm_animal, inheritance_rhino_mammal,
+        inheritance_vine_plant, inheritance_ent_plant, similarity_monkey_human,
+        similarity_chimp_human, similarity_monkey_chimp, similarity_earthworm_snake,
+        similarity_triceratops_rhino, similarity_vine_snake, similarity_ent_human]
 # fmt: on
 
 

tests/integration/remote_das_info.py

@@ -1,2 +1,3 @@
+#remote_das_host = "localhost"
 remote_das_host = "45.63.85.59"
 remote_das_port = 8080

tests/integration/test_das_query_api.py

@@ -0,0 +1,73 @@
+import pytest
+
+from hyperon_das import DistributedAtomSpace
+
+from .helpers import (
+    MettaAnimalBaseHandlesCollection,
+    _db_down,
+    _db_up,
+    cleanup,
+    load_metta_animals_base,
+    mongo_port,
+    redis_port,
+)
+from .remote_das_info import remote_das_host, remote_das_port
+
+das_instance = {}
+
+
+class TestDASQueryAPI:
+    """Test query methods in DAS API with integration of DAS and its various AtomDB adapters"""
+
+    @classmethod
+    def setup_class(cls):
+        _db_up()
+
+        das_instance["local_ram"] = DistributedAtomSpace(query_engine='local', atomdb='ram')
+        load_metta_animals_base(das_instance["local_ram"])
+
+        das_instance["local_redis_mongo"] = DistributedAtomSpace(
+            query_engine='local',
+            atomdb='redis_mongo',
+            mongo_port=mongo_port,
+            mongo_username='dbadmin',
+            mongo_password='dassecret',
+            redis_port=redis_port,
+            redis_cluster=False,
+            redis_ssl=False,
+        )
+        load_metta_animals_base(das_instance["local_redis_mongo"])
+        das_instance["local_redis_mongo"].commit_changes()
+
+        das_instance["remote"] = DistributedAtomSpace(
+            query_engine='remote', host=remote_das_host, port=remote_das_port
+        )
+
+    @classmethod
+    def teardown_class(cls):
+        _db_down()
+
+    @pytest.fixture(scope="session", autouse=True)
+    def _cleanup(self, request):
+        return cleanup(request)
+
+    def test_count_atoms(self):
+        for key, das in das_instance.items():
+            count = das.count_atoms({})
+            assert count["atom_count"] == 66
+
+    def test_get_atom(self):
+        for key, das in das_instance.items():
+            node_list = []
+            for handle in MettaAnimalBaseHandlesCollection.node_handles:
+                atom = das.get_atom(handle)
+                assert atom["handle"] == handle
+                node_list.append(atom)
+            link_list = []
+            for handle in MettaAnimalBaseHandlesCollection.link_handles:
+                atom = das.get_atom(handle)
+                assert atom["handle"] == handle
+                link_list.append(atom)
+            if key != "remote":
+                assert node_list == das.get_atoms(MettaAnimalBaseHandlesCollection.node_handles)
+                assert link_list == das.get_atoms(MettaAnimalBaseHandlesCollection.link_handles)