Merge pull request #54 from singnet/senna-43-4

[#43] Create integration test for redis-mongo DB
singnet · Dec 7, 2023 · 3e850c5 · 3e850c5
2 parents a5d68c3 + a5d32f3
commit 3e850c5
Show file tree

Hide file tree

Showing 6 changed files with 225 additions and 1 deletion.
diff --git a/hyperon_das_atomdb/adapters/redis_mongo_db.py b/hyperon_das_atomdb/adapters/redis_mongo_db.py
@@ -4,6 +4,7 @@
 
 from pymongo import MongoClient
 from pymongo.database import Database
+from pymongo.errors import BulkWriteError
 from redis import Redis
 from redis.cluster import RedisCluster
 import sys
@@ -580,7 +581,12 @@ def commit(self) -> None:
         for key, (collection, buffer) in self.mongo_bulk_insertion_buffer.items():
             if buffer:
                 documents = [d.base for d in buffer]
-                collection.insert_many(documents, ordered=False)
+                try:
+                    collection.insert_many(documents, ordered=False)
+                except BulkWriteError as exception:
+                    for error in exception.details["writeErrors"]:
+                        if error["code"] != 11000: # duplicate insertion error
+                            raise exception
                 if key == MongoCollectionNames.NODES:
                     self._update_node_index(documents)
                 elif key == MongoCollectionNames.ATOM_TYPES:

diff --git a/tests/integration/adapters/test_redis_mongo.py b/tests/integration/adapters/test_redis_mongo.py
@@ -0,0 +1,125 @@
+import subprocess
+import pytest
+import os
+from hyperon_das_atomdb.adapters import RedisMongoDB
+
+redis_port = "15926"
+mongo_port = "15927"
+scripts_path = "./tests/integration/scripts/"
+devnull = open(os.devnull, 'w')
+
+DAS_MONGODB_HOSTNAME = os.environ.get("DAS_MONGODB_HOSTNAME")
+DAS_MONGODB_PORT = os.environ.get("DAS_MONGODB_PORT")
+DAS_MONGODB_USERNAME = os.environ.get("DAS_MONGODB_USERNAME")
+DAS_MONGODB_PASSWORD = os.environ.get("DAS_MONGODB_PASSWORD")
+DAS_REDIS_HOSTNAME = os.environ.get("DAS_REDIS_HOSTNAME")
+DAS_REDIS_PORT = os.environ.get("DAS_REDIS_PORT")
+DAS_REDIS_USERNAME = os.environ.get("DAS_REDIS_USERNAME")
+DAS_REDIS_PASSWORD = os.environ.get("DAS_REDIS_PASSWORD")
+DAS_USE_REDIS_CLUSTER = os.environ.get("DAS_USE_REDIS_CLUSTER")
+DAS_USE_REDIS_SSL = os.environ.get("DAS_USE_REDIS_SSL")
+
+os.environ["DAS_MONGODB_HOSTNAME"] = "localhost"
+os.environ["DAS_MONGODB_PORT"] = mongo_port
+os.environ["DAS_MONGODB_USERNAME"] = "dbadmin"
+os.environ["DAS_MONGODB_PASSWORD"] = "dassecret"
+os.environ["DAS_REDIS_HOSTNAME"] = "localhost"
+os.environ["DAS_REDIS_PORT"] = redis_port
+os.environ["DAS_REDIS_USERNAME"] = ""
+os.environ["DAS_REDIS_PASSWORD"] = ""
+os.environ["DAS_USE_REDIS_CLUSTER"] = "false"
+os.environ["DAS_USE_REDIS_SSL"] = "false"
+
+def _db_up():
+    subprocess.call(["bash", f"{scripts_path}/redis-up.sh", redis_port], stdout=devnull, stderr=devnull)
+    subprocess.call(["bash", f"{scripts_path}/mongo-up.sh", mongo_port], stdout=devnull, stderr=devnull)
+
+def _db_down():
+    subprocess.call(["bash", f"{scripts_path}/redis-down.sh", redis_port], stdout=devnull, stderr=devnull)
+    subprocess.call(["bash", f"{scripts_path}/mongo-down.sh", mongo_port], stdout=devnull, stderr=devnull)
+
+@pytest.fixture(scope="session", autouse=True)
+def cleanup(request):
+
+    def restore_environment():
+        if DAS_MONGODB_HOSTNAME:
+            os.environ["DAS_MONGODB_HOSTNAME"] = DAS_MONGODB_HOSTNAME
+        if DAS_MONGODB_PORT:
+            os.environ["DAS_MONGODB_PORT"] = DAS_MONGODB_PORT
+        if DAS_MONGODB_USERNAME:
+            os.environ["DAS_MONGODB_USERNAME"] = DAS_MONGODB_USERNAME
+        if DAS_MONGODB_PASSWORD:
+            os.environ["DAS_MONGODB_PASSWORD"] = DAS_MONGODB_PASSWORD
+        if DAS_REDIS_HOSTNAME:
+            os.environ["DAS_REDIS_HOSTNAME"] = DAS_REDIS_HOSTNAME
+        if DAS_REDIS_PORT:
+            os.environ["DAS_REDIS_PORT"] = DAS_REDIS_PORT
+        if DAS_REDIS_USERNAME:
+            os.environ["DAS_REDIS_USERNAME"] = DAS_REDIS_USERNAME
+        if DAS_REDIS_PASSWORD:
+            os.environ["DAS_REDIS_PASSWORD"] = DAS_REDIS_PASSWORD
+        if DAS_USE_REDIS_CLUSTER:
+            os.environ["DAS_USE_REDIS_CLUSTER"] = DAS_USE_REDIS_CLUSTER
+        if DAS_USE_REDIS_SSL:
+            os.environ["DAS_USE_REDIS_SSL"] = DAS_USE_REDIS_SSL
+
+    def enforce_containers_removal():
+        _db_down()
+
+    request.addfinalizer(restore_environment)
+    request.addfinalizer(enforce_containers_removal)
+
+class TestRedisMongo:
+
+    def _add_atoms(self, db: RedisMongoDB):
+        db.add_node({"type": "Concept", "name": "human"})
+        db.add_node({"type": "Concept", "name": "monkey"})
+        db.add_node({"type": "Concept", "name": "chimp"})
+        db.add_node({"type": "Concept", "name": "mammal"})
+        db.add_node({"type": "Concept", "name": "reptile"})
+        db.add_node({"type": "Concept", "name": "snake"})
+        db.add_node({"type": "Concept", "name": "dinosaur"})
+        db.add_node({"type": "Concept", "name": "triceratops"})
+        db.add_node({"type": "Concept", "name": "earthworm"})
+        db.add_node({"type": "Concept", "name": "rhino"})
+        db.add_node({"type": "Concept", "name": "vine"})
+        db.add_node({"type": "Concept", "name": "ent"})
+        db.add_node({"type": "Concept", "name": "animal"})
+        db.add_node({"type": "Concept", "name": "plant"})
+
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "monkey"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "chimp"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "monkey"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "earthworm"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "rhino"}, {"type": "Concept", "name": "triceratops"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "vine"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "ent"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "human"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "human"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "chimp"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "earthworm"}, {"type": "Concept", "name": "snake"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "triceratops"}, {"type": "Concept", "name": "rhino"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "vine"}, {"type": "Concept", "name": "snake"},]}) 
+        db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "ent"}, {"type": "Concept", "name": "human"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "mammal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "mammal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "mammal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "mammal"}, {"type": "Concept", "name": "animal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "reptile"}, {"type": "Concept", "name": "animal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "reptile"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "dinosaur"}, {"type": "Concept", "name": "reptile"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "triceratops"}, {"type": "Concept", "name": "dinosaur"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "earthworm"}, {"type": "Concept", "name": "animal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "rhino"}, {"type": "Concept", "name": "mammal"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "vine"}, {"type": "Concept", "name": "plant"},]}) 
+        db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "ent"}, {"type": "Concept", "name": "plant"},]})
+
+    def test_commit(self):
+        _db_up()
+        db = RedisMongoDB()
+        assert db.count_atoms() == (0, 0)
+        self._add_atoms(db)
+        assert db.count_atoms() == (0, 0)
+        db.commit()
+        assert db.count_atoms() == (14, 26)
+        _db_down()
diff --git a/tests/integration/scripts/mongo-down.sh b/tests/integration/scripts/mongo-down.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ -z "$1" ]
+then
+    echo "Usage: mongo-down.sh PORT"
+    exit 1
+else
+    PORT=$1
+fi
+
+echo "Destroying MongoDB container on port $PORT"
+
+docker stop mongo_$PORT && docker rm mongo_$PORT && docker volume rm mongodbdata_$PORT >& /dev/null
diff --git a/tests/integration/scripts/mongo-up.sh b/tests/integration/scripts/mongo-up.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [ -z "$1" ]
+then
+    echo "Usage: mongo-up.sh PORT"
+    exit 1
+else
+    PORT=$1
+fi
+
+echo "Starting MongoDB on port $PORT"
+
+docker stop mongo_$PORT >& /dev/null
+docker rm mongo_$PORT >& /dev/null
+docker volume rm mongodbdata_$PORT >& /dev/null
+
+sleep 1
+docker run \
+    --detach \
+    --name mongo_$PORT \
+    --env MONGO_INITDB_ROOT_USERNAME="dbadmin" \
+    --env MONGO_INITDB_ROOT_PASSWORD="dassecret" \
+    --env TZ=${TZ} \
+    --network="host" \
+    --volume /tmp:/tmp \
+    --volume /mnt:/mnt \
+    --volume mongodbdata_$PORT:/data/db \
+    mongo:latest \
+    mongod --port $PORT
diff --git a/tests/integration/scripts/redis-down.sh b/tests/integration/scripts/redis-down.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ -z "$1" ]
+then
+    echo "Usage: redis-down.sh PORT"
+    exit 1
+else
+    PORT=$1
+fi
+
+echo "Destroying Redis container on port $PORT"
+
+docker stop redis_$PORT && docker rm redis_$PORT
diff --git a/tests/integration/scripts/redis-up.sh b/tests/integration/scripts/redis-up.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+if [ -z "$1" ]
+then
+    echo "Usage: redis-up.sh PORT"
+    exit 1
+else
+    PORT=$1
+    CLUSTER_ENABLED="no"
+fi
+
+CONFIG_FILE="/tmp/redis_$PORT.conf"
+cp ./redis.conf $CONFIG_FILE
+
+cat <<EOM >$CONFIG_FILE
+cluster-config-file redis-cluster-state.conf
+cluster-node-timeout 30000
+appendonly yes
+protected-mode no
+EOM
+
+echo "cluster-enabled $CLUSTER_ENABLED" >> $CONFIG_FILE
+echo "port $PORT" >> $CONFIG_FILE
+
+echo "Starting Redis on port $PORT"
+
+docker stop redis_$PORT >& /dev/null
+docker rm redis_$PORT >& /dev/null
+
+docker run \
+    --detach \
+    --name redis_$PORT \
+    --env TZ=${TZ} \
+    --network="host" \
+    --volume /tmp:/tmp \
+    --volume /mnt:/mnt \
+    redis/redis-stack-server:latest \
+    redis-server $CONFIG_FILE