From a5d32f3a68fea4fe0e75160c2c3bd67d325260a4 Mon Sep 17 00:00:00 2001 From: Andre Senna <“andre.senna@gmail.com”> Date: Thu, 7 Dec 2023 12:31:19 -0300 Subject: [PATCH] Create integration test for redis-mongo DB --- hyperon_das_atomdb/adapters/redis_mongo_db.py | 8 +- .../integration/adapters/test_redis_mongo.py | 125 ++++++++++++++++++ tests/integration/scripts/mongo-down.sh | 13 ++ tests/integration/scripts/mongo-up.sh | 29 ++++ tests/integration/scripts/redis-down.sh | 13 ++ tests/integration/scripts/redis-up.sh | 38 ++++++ 6 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 tests/integration/adapters/test_redis_mongo.py create mode 100755 tests/integration/scripts/mongo-down.sh create mode 100755 tests/integration/scripts/mongo-up.sh create mode 100755 tests/integration/scripts/redis-down.sh create mode 100755 tests/integration/scripts/redis-up.sh diff --git a/hyperon_das_atomdb/adapters/redis_mongo_db.py b/hyperon_das_atomdb/adapters/redis_mongo_db.py index 876c2fd9..b4968021 100644 --- a/hyperon_das_atomdb/adapters/redis_mongo_db.py +++ b/hyperon_das_atomdb/adapters/redis_mongo_db.py @@ -4,6 +4,7 @@ from pymongo import MongoClient from pymongo.database import Database +from pymongo.errors import BulkWriteError from redis import Redis from redis.cluster import RedisCluster import sys @@ -580,7 +581,12 @@ def commit(self) -> None: for key, (collection, buffer) in self.mongo_bulk_insertion_buffer.items(): if buffer: documents = [d.base for d in buffer] - collection.insert_many(documents, ordered=False) + try: + collection.insert_many(documents, ordered=False) + except BulkWriteError as exception: + for error in exception.details["writeErrors"]: + if error["code"] != 11000: # duplicate insertion error + raise exception if key == MongoCollectionNames.NODES: self._update_node_index(documents) elif key == MongoCollectionNames.ATOM_TYPES: diff --git a/tests/integration/adapters/test_redis_mongo.py b/tests/integration/adapters/test_redis_mongo.py new file mode 100644 index 00000000..c75e5082 --- /dev/null +++ b/tests/integration/adapters/test_redis_mongo.py @@ -0,0 +1,125 @@ +import subprocess +import pytest +import os +from hyperon_das_atomdb.adapters import RedisMongoDB + +redis_port = "15926" +mongo_port = "15927" +scripts_path = "./tests/integration/scripts/" +devnull = open(os.devnull, 'w') + +DAS_MONGODB_HOSTNAME = os.environ.get("DAS_MONGODB_HOSTNAME") +DAS_MONGODB_PORT = os.environ.get("DAS_MONGODB_PORT") +DAS_MONGODB_USERNAME = os.environ.get("DAS_MONGODB_USERNAME") +DAS_MONGODB_PASSWORD = os.environ.get("DAS_MONGODB_PASSWORD") +DAS_REDIS_HOSTNAME = os.environ.get("DAS_REDIS_HOSTNAME") +DAS_REDIS_PORT = os.environ.get("DAS_REDIS_PORT") +DAS_REDIS_USERNAME = os.environ.get("DAS_REDIS_USERNAME") +DAS_REDIS_PASSWORD = os.environ.get("DAS_REDIS_PASSWORD") +DAS_USE_REDIS_CLUSTER = os.environ.get("DAS_USE_REDIS_CLUSTER") +DAS_USE_REDIS_SSL = os.environ.get("DAS_USE_REDIS_SSL") + +os.environ["DAS_MONGODB_HOSTNAME"] = "localhost" +os.environ["DAS_MONGODB_PORT"] = mongo_port +os.environ["DAS_MONGODB_USERNAME"] = "dbadmin" +os.environ["DAS_MONGODB_PASSWORD"] = "dassecret" +os.environ["DAS_REDIS_HOSTNAME"] = "localhost" +os.environ["DAS_REDIS_PORT"] = redis_port +os.environ["DAS_REDIS_USERNAME"] = "" +os.environ["DAS_REDIS_PASSWORD"] = "" +os.environ["DAS_USE_REDIS_CLUSTER"] = "false" +os.environ["DAS_USE_REDIS_SSL"] = "false" + +def _db_up(): + subprocess.call(["bash", f"{scripts_path}/redis-up.sh", redis_port], stdout=devnull, stderr=devnull) + subprocess.call(["bash", f"{scripts_path}/mongo-up.sh", mongo_port], stdout=devnull, stderr=devnull) + +def _db_down(): + subprocess.call(["bash", f"{scripts_path}/redis-down.sh", redis_port], stdout=devnull, stderr=devnull) + subprocess.call(["bash", f"{scripts_path}/mongo-down.sh", mongo_port], stdout=devnull, stderr=devnull) + +@pytest.fixture(scope="session", autouse=True) +def cleanup(request): + + def restore_environment(): + if DAS_MONGODB_HOSTNAME: + os.environ["DAS_MONGODB_HOSTNAME"] = DAS_MONGODB_HOSTNAME + if DAS_MONGODB_PORT: + os.environ["DAS_MONGODB_PORT"] = DAS_MONGODB_PORT + if DAS_MONGODB_USERNAME: + os.environ["DAS_MONGODB_USERNAME"] = DAS_MONGODB_USERNAME + if DAS_MONGODB_PASSWORD: + os.environ["DAS_MONGODB_PASSWORD"] = DAS_MONGODB_PASSWORD + if DAS_REDIS_HOSTNAME: + os.environ["DAS_REDIS_HOSTNAME"] = DAS_REDIS_HOSTNAME + if DAS_REDIS_PORT: + os.environ["DAS_REDIS_PORT"] = DAS_REDIS_PORT + if DAS_REDIS_USERNAME: + os.environ["DAS_REDIS_USERNAME"] = DAS_REDIS_USERNAME + if DAS_REDIS_PASSWORD: + os.environ["DAS_REDIS_PASSWORD"] = DAS_REDIS_PASSWORD + if DAS_USE_REDIS_CLUSTER: + os.environ["DAS_USE_REDIS_CLUSTER"] = DAS_USE_REDIS_CLUSTER + if DAS_USE_REDIS_SSL: + os.environ["DAS_USE_REDIS_SSL"] = DAS_USE_REDIS_SSL + + def enforce_containers_removal(): + _db_down() + + request.addfinalizer(restore_environment) + request.addfinalizer(enforce_containers_removal) + +class TestRedisMongo: + + def _add_atoms(self, db: RedisMongoDB): + db.add_node({"type": "Concept", "name": "human"}) + db.add_node({"type": "Concept", "name": "monkey"}) + db.add_node({"type": "Concept", "name": "chimp"}) + db.add_node({"type": "Concept", "name": "mammal"}) + db.add_node({"type": "Concept", "name": "reptile"}) + db.add_node({"type": "Concept", "name": "snake"}) + db.add_node({"type": "Concept", "name": "dinosaur"}) + db.add_node({"type": "Concept", "name": "triceratops"}) + db.add_node({"type": "Concept", "name": "earthworm"}) + db.add_node({"type": "Concept", "name": "rhino"}) + db.add_node({"type": "Concept", "name": "vine"}) + db.add_node({"type": "Concept", "name": "ent"}) + db.add_node({"type": "Concept", "name": "animal"}) + db.add_node({"type": "Concept", "name": "plant"}) + + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "monkey"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "chimp"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "monkey"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "earthworm"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "rhino"}, {"type": "Concept", "name": "triceratops"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "vine"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "ent"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "human"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "human"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "chimp"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "earthworm"}, {"type": "Concept", "name": "snake"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "triceratops"}, {"type": "Concept", "name": "rhino"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "vine"}, {"type": "Concept", "name": "snake"},]}) + db.add_link({ "type": "Similarity", "targets": [ {"type": "Concept", "name": "ent"}, {"type": "Concept", "name": "human"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "human"}, {"type": "Concept", "name": "mammal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "monkey"}, {"type": "Concept", "name": "mammal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "chimp"}, {"type": "Concept", "name": "mammal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "mammal"}, {"type": "Concept", "name": "animal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "reptile"}, {"type": "Concept", "name": "animal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "snake"}, {"type": "Concept", "name": "reptile"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "dinosaur"}, {"type": "Concept", "name": "reptile"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "triceratops"}, {"type": "Concept", "name": "dinosaur"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "earthworm"}, {"type": "Concept", "name": "animal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "rhino"}, {"type": "Concept", "name": "mammal"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "vine"}, {"type": "Concept", "name": "plant"},]}) + db.add_link({ "type": "Inheritance", "targets": [ {"type": "Concept", "name": "ent"}, {"type": "Concept", "name": "plant"},]}) + + def test_commit(self): + _db_up() + db = RedisMongoDB() + assert db.count_atoms() == (0, 0) + self._add_atoms(db) + assert db.count_atoms() == (0, 0) + db.commit() + assert db.count_atoms() == (14, 26) + _db_down() diff --git a/tests/integration/scripts/mongo-down.sh b/tests/integration/scripts/mongo-down.sh new file mode 100755 index 00000000..b4429e2b --- /dev/null +++ b/tests/integration/scripts/mongo-down.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ -z "$1" ] +then + echo "Usage: mongo-down.sh PORT" + exit 1 +else + PORT=$1 +fi + +echo "Destroying MongoDB container on port $PORT" + +docker stop mongo_$PORT && docker rm mongo_$PORT && docker volume rm mongodbdata_$PORT >& /dev/null diff --git a/tests/integration/scripts/mongo-up.sh b/tests/integration/scripts/mongo-up.sh new file mode 100755 index 00000000..f5896354 --- /dev/null +++ b/tests/integration/scripts/mongo-up.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ -z "$1" ] +then + echo "Usage: mongo-up.sh PORT" + exit 1 +else + PORT=$1 +fi + +echo "Starting MongoDB on port $PORT" + +docker stop mongo_$PORT >& /dev/null +docker rm mongo_$PORT >& /dev/null +docker volume rm mongodbdata_$PORT >& /dev/null + +sleep 1 +docker run \ + --detach \ + --name mongo_$PORT \ + --env MONGO_INITDB_ROOT_USERNAME="dbadmin" \ + --env MONGO_INITDB_ROOT_PASSWORD="dassecret" \ + --env TZ=${TZ} \ + --network="host" \ + --volume /tmp:/tmp \ + --volume /mnt:/mnt \ + --volume mongodbdata_$PORT:/data/db \ + mongo:latest \ + mongod --port $PORT diff --git a/tests/integration/scripts/redis-down.sh b/tests/integration/scripts/redis-down.sh new file mode 100755 index 00000000..72d856cf --- /dev/null +++ b/tests/integration/scripts/redis-down.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ -z "$1" ] +then + echo "Usage: redis-down.sh PORT" + exit 1 +else + PORT=$1 +fi + +echo "Destroying Redis container on port $PORT" + +docker stop redis_$PORT && docker rm redis_$PORT diff --git a/tests/integration/scripts/redis-up.sh b/tests/integration/scripts/redis-up.sh new file mode 100755 index 00000000..34658197 --- /dev/null +++ b/tests/integration/scripts/redis-up.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +if [ -z "$1" ] +then + echo "Usage: redis-up.sh PORT" + exit 1 +else + PORT=$1 + CLUSTER_ENABLED="no" +fi + +CONFIG_FILE="/tmp/redis_$PORT.conf" +cp ./redis.conf $CONFIG_FILE + +cat <$CONFIG_FILE +cluster-config-file redis-cluster-state.conf +cluster-node-timeout 30000 +appendonly yes +protected-mode no +EOM + +echo "cluster-enabled $CLUSTER_ENABLED" >> $CONFIG_FILE +echo "port $PORT" >> $CONFIG_FILE + +echo "Starting Redis on port $PORT" + +docker stop redis_$PORT >& /dev/null +docker rm redis_$PORT >& /dev/null + +docker run \ + --detach \ + --name redis_$PORT \ + --env TZ=${TZ} \ + --network="host" \ + --volume /tmp:/tmp \ + --volume /mnt:/mnt \ + redis/redis-stack-server:latest \ + redis-server $CONFIG_FILE