From c5f1831fef9367367adad757eff1e3824c14d17f Mon Sep 17 00:00:00 2001 From: John Andersen Date: Mon, 25 Mar 2024 19:04:56 +0000 Subject: [PATCH] source: mongodb: Remove unpublished plugin Related: https://github.com/intel/dffml/pull/1481 Signed-off-by: John Andersen --- source/mongodb/.coveragerc | 13 -- source/mongodb/.gitignore | 21 --- source/mongodb/LICENSE | 21 --- source/mongodb/MANIFEST.in | 3 - source/mongodb/README.rst | 2 - .../mongodb/dffml_source_mongodb/__init__.py | 0 source/mongodb/dffml_source_mongodb/source.py | 93 ---------- .../dffml_source_mongodb/util/__init__.py | 0 .../util/mongodb_docker.py | 161 ------------------ .../mongodb/dffml_source_mongodb/version.py | 1 - source/mongodb/entry_points.txt | 2 - source/mongodb/pyproject.toml | 22 --- source/mongodb/setup.cfg | 41 ----- source/mongodb/setup.py | 8 - source/mongodb/tests/__init__.py | 0 source/mongodb/tests/test_source.py | 56 ------ 16 files changed, 444 deletions(-) delete mode 100644 source/mongodb/.coveragerc delete mode 100644 source/mongodb/.gitignore delete mode 100644 source/mongodb/LICENSE delete mode 100644 source/mongodb/MANIFEST.in delete mode 100644 source/mongodb/README.rst delete mode 100644 source/mongodb/dffml_source_mongodb/__init__.py delete mode 100644 source/mongodb/dffml_source_mongodb/source.py delete mode 100644 source/mongodb/dffml_source_mongodb/util/__init__.py delete mode 100644 source/mongodb/dffml_source_mongodb/util/mongodb_docker.py delete mode 100644 source/mongodb/dffml_source_mongodb/version.py delete mode 100644 source/mongodb/entry_points.txt delete mode 100644 source/mongodb/pyproject.toml delete mode 100644 source/mongodb/setup.cfg delete mode 100644 source/mongodb/setup.py delete mode 100644 source/mongodb/tests/__init__.py delete mode 100644 source/mongodb/tests/test_source.py diff --git a/source/mongodb/.coveragerc b/source/mongodb/.coveragerc deleted file mode 100644 index 66a98aa5ab..0000000000 --- a/source/mongodb/.coveragerc +++ /dev/null @@ -1,13 +0,0 @@ -[run] -source = - dffml_source_mongodb - tests -branch = True - -[report] -exclude_lines = - no cov - no qa - noqa - pragma: no cover - if __name__ == .__main__.: diff --git a/source/mongodb/.gitignore b/source/mongodb/.gitignore deleted file mode 100644 index 3af0b3e081..0000000000 --- a/source/mongodb/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -*.log -*.pyc -.cache/ -.coverage -.idea/ -.vscode/ -*.egg-info/ -build/ -dist/ -docs/build/ -venv/ -wheelhouse/ -*.egss -.mypy_cache/ -*.swp -.venv/ -.eggs/ -*.modeldir -*.db -htmlcov/ -built_html_docs/ diff --git a/source/mongodb/LICENSE b/source/mongodb/LICENSE deleted file mode 100644 index 276b9945c4..0000000000 --- a/source/mongodb/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2021 Intel - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/source/mongodb/MANIFEST.in b/source/mongodb/MANIFEST.in deleted file mode 100644 index 6d6d7abb8f..0000000000 --- a/source/mongodb/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.rst -include LICENSE -recursive-include dffml_source_mongodb * diff --git a/source/mongodb/README.rst b/source/mongodb/README.rst deleted file mode 100644 index 5118d8716d..0000000000 --- a/source/mongodb/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -DFFML Source for MongoDB -======================== diff --git a/source/mongodb/dffml_source_mongodb/__init__.py b/source/mongodb/dffml_source_mongodb/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/mongodb/dffml_source_mongodb/source.py b/source/mongodb/dffml_source_mongodb/source.py deleted file mode 100644 index 01621851e7..0000000000 --- a/source/mongodb/dffml_source_mongodb/source.py +++ /dev/null @@ -1,93 +0,0 @@ -import urllib.parse -from typing import AsyncIterator, Dict, List - -from dffml.base import BaseConfig -from dffml.record import Record -from dffml.source.source import BaseSourceContext, BaseSource -from dffml.util.cli.arg import Arg -from dffml.util.entrypoint import entrypoint -from dffml.base import config - - -import motor.motor_asyncio - - -@config -class MongoDBSourceConfig: - uri: str - db: str = None - collection: str = None - tlsInsecure: bool = False - log_collection_names: bool = False - - def __post_init__(self): - uri = urllib.parse.urlparse(self.uri) - if uri.path: - self.db = uri.path[1:] - - -# TODO Investigate use of -# https://pymongo.readthedocs.io/en/3.12.0/api/pymongo/client_session.html#pymongo.client_session.ClientSession -# for Context. -class MongoDBSourceContext(BaseSourceContext): - async def update(self, record): - self.logger.debug("update: %s: %r", record.key, record.export()) - await self.parent.collection.replace_one( - {"_id": record.key}, - {"_id": record.key, **record.export()}, - upsert=True, - ) - - def document_to_record(self, document, key=None): - self.logger.debug("document: %r", document) - if document is None: - if key is None: - raise ValueError("Cannot create empty record with no key") - return Record(key) - if "key" in document: - key = document["key"] - del document["key"] - else: - key = document["_id"] - del document["_id"] - if "features" in document: - return Record(key, data=document) - else: - return Record(key, data={"features": document}) - - async def records(self) -> AsyncIterator[Record]: - async for document in self.parent.collection.find(): - yield self.document_to_record(document) - - async def record(self, key: str) -> Record: - document = await self.parent.collection.find_one({"_id": key}) - return self.document_to_record(document, key=key) - - -@entrypoint("mongodb") -class MongoDBSource(BaseSource): - """ - Stores records ... somewhere! (skeleton template is in memory) - """ - - CONFIG = MongoDBSourceConfig - CONTEXT = MongoDBSourceContext - - def __init__(self, config: BaseConfig) -> None: - super().__init__(config) - self.client = None - - async def __aenter__(self): - self.client = motor.motor_asyncio.AsyncIOMotorClient(self.config.uri, - tlsInsecure=self.config.tlsInsecure) - self.db = self.client[self.config.db] - # Thought: Plugins as dataflows. Is a method call an event? Is it an - # input? - if self.config.log_collection_names: - self.logger.info("Collection names: %r", await self.db.list_collection_names()) - self.collection = self.db[self.config.collection] - self.logger.info("Collection options: %r", await self.collection.options()) - return self - - async def __aexec__(self, _exc_type, _exc_value, _traceback): - self.client = None diff --git a/source/mongodb/dffml_source_mongodb/util/__init__.py b/source/mongodb/dffml_source_mongodb/util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/mongodb/dffml_source_mongodb/util/mongodb_docker.py b/source/mongodb/dffml_source_mongodb/util/mongodb_docker.py deleted file mode 100644 index dbaa796922..0000000000 --- a/source/mongodb/dffml_source_mongodb/util/mongodb_docker.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -import json -import time -import atexit -import socket -import pathlib -import logging -import tempfile -import unittest -import subprocess -from contextlib import contextmanager -from typing import Optional - -import docker - -LOGGER = logging.getLogger(__package__) - -logging.basicConfig(level=logging.DEBUG) - -DOCKER_IMAGE = "mongo:4" -# MongoDB daemons default listing port -DEFAULT_PORT = 27017 -# Environment variables passed to MongoDB container -DOCKER_ENV = { - "MONGO_INITDB_ROOT_USERNAME": "mongoadmin", - "MONGO_INITDB_ROOT_PASSWORD": "secret", -} -DOCKER_NA: str = "Failed to connect to docker daemon" -DOCKER_AVAILABLE: bool = False -try: - DOCKER_CLIENT: docker.DockerClient = docker.from_env() - DOCKER_AVAILABLE = DOCKER_CLIENT.ping() - DOCKER_CLIENT.close() -except: - pass - - -class MongoDBFailedToStart(Exception): - pass # pragma: no cov - - -def check_connection(addr: str, port: int, *, timeout: float = 0.1) -> bool: - """ - Attempt to make a TCP connection. Return if a connection was made in - less than ``timeout`` seconds. Return True if a connection is made within - the timeout. - """ - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(float(timeout)) - try: - s.connect((addr, port)) - except Exception as error: - return False - return True - - -def mkcleanup(docker_client, container): - """ - Create a function which will remove the temporary file and stop the - container. The function will register itself with the :py:`atexit` module to - ensure that the container is stopped before Python exits. It will unregister - itself whenever it is called. - """ - func = None - - def cleanup(): - atexit.unregister(func) - try: - container.stop() - container.wait() - except: - pass - docker_client.close() - - func = cleanup - atexit.register(func) - return cleanup - - -@contextmanager -def mongodb(*, js_setup: Optional[str] = None): - """ - Start a MongoDB container and yield the IP of the container once ready for - connections. ``js_setup`` should be the .sql file used to initialize the - database. - """ - if not DOCKER_AVAILABLE: - raise unittest.SkipTest("Need docker to run MongoDB") - - docker_client: docker.DockerClient = docker.from_env() - with tempfile.TemporaryDirectory() as tempdir: - # Volumes to mount - volumes = {} - # Dump out JavaScript initialization file - if js_setup is not None: - js_setup_path = pathlib.Path(tempdir, "dump.js") - js_setup_path.write_text(js_setup) - js_setup_path.chmod(0o555) - volumes[js_setup_path.resolve()] = { - "bind": "/docker-entrypoint-initdb.d/dump.js" - } - # Tell the docker daemon to start MongoDB - LOGGER.debug("Starting MongoDB...") - container = docker_client.containers.run( - DOCKER_IMAGE, - environment=DOCKER_ENV, - detach=True, - auto_remove=True, - volumes=volumes, - ) - # Sometimes very bad things happen, this ensures that the container will - # be cleaned up on process exit no matter what - cleanup = mkcleanup(docker_client, container) - try: - # Get the IP from the docker daemon - inspect = docker_client.api.inspect_container(container.id) - container_ip = inspect["NetworkSettings"]["IPAddress"] - # Wait until MongoDB reports it's ready for connections - container_start_time = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - ready = False - for line in container.logs(stream=True, follow=True): - now_time = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - LOGGER.debug( - "MongoDB log (%0.02f seconds): %s", - (now_time - container_start_time), - line.decode(errors="ignore").strip(), - ) - if not line.startswith(b"{"): - continue - log_entry = json.loads(line.decode()) - if ( - log_entry["c"] == "NETWORK" - and log_entry["ctx"] == "listener" - and log_entry["msg"] == "Waiting for connections" - ): - ready = True - break - if not ready: - raise MongoDBFailedToStart( - 'Never saw "Waiting for connections"' - ) - # Ensure that we can make a connection - start_time = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - max_timeout = float(os.getenv("MONGODB_START_TIMEOUT", "600")) - LOGGER.debug( - "Attempting to connect to MongoDB: Timeout of %d seconds", - max_timeout, - ) - while not check_connection(container_ip, DEFAULT_PORT): - end_time = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - if (end_time - start_time) >= max_timeout: - raise MongoDBFailedToStart("Timed out waiting for MongoDB") - end_time = time.clock_gettime(time.CLOCK_MONOTONIC_RAW) - LOGGER.debug( - "MongoDB running: Took %0.02f seconds", - end_time - container_start_time, - ) - # Yield IP of container to caller - yield container_ip - finally: - cleanup() diff --git a/source/mongodb/dffml_source_mongodb/version.py b/source/mongodb/dffml_source_mongodb/version.py deleted file mode 100644 index 901e5110b2..0000000000 --- a/source/mongodb/dffml_source_mongodb/version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = "0.0.1" diff --git a/source/mongodb/entry_points.txt b/source/mongodb/entry_points.txt deleted file mode 100644 index bda54d210d..0000000000 --- a/source/mongodb/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[dffml.source] -mongodb = dffml_source_mongodb.source:MongoDBSource diff --git a/source/mongodb/pyproject.toml b/source/mongodb/pyproject.toml deleted file mode 100644 index 17b1235941..0000000000 --- a/source/mongodb/pyproject.toml +++ /dev/null @@ -1,22 +0,0 @@ -requires = ["setuptools>=44", "wheel", "setuptools_scm[toml]>=3.4.3"] -build-backend = "setuptools.build_meta" - -[tool.setuptools_scm] - -[tool.black] -exclude = ''' -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - ) -) -''' diff --git a/source/mongodb/setup.cfg b/source/mongodb/setup.cfg deleted file mode 100644 index 609ea01a63..0000000000 --- a/source/mongodb/setup.cfg +++ /dev/null @@ -1,41 +0,0 @@ -[metadata] -name = dffml-source-mongodb -description = DFFML source dffml-source-mongodb -version = attr: dffml_source_mongodb.version.VERSION -long_description = file: README.rst -author = John Andersen -author_email = johnandersenpdx@gmail.com -maintainer = John Andersen -maintainer_email = johnandersenpdx@gmail.com -url = https://github.com/dffml/dffml-source-mongodb -license = MIT -keywords = dffml -classifiers = - Development Status :: 3 - Alpha - Intended Audience :: Developers - License :: OSI Approved :: MIT License - Natural Language :: English - Operating System :: OS Independent - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.7 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - -[options] -zip_safe = False -include_package_data = True -packages = find: -entry_points = file: entry_points.txt -install_requires = - dffml>=0.4.0 - motor>=2.5.1 - -[options.extras_require] -dev = - coverage - codecov - sphinx - twine - black==19.10b0 - importlib_metadata>=4.8.1;python_version<"3.8" - docker>=4.0.2 diff --git a/source/mongodb/setup.py b/source/mongodb/setup.py deleted file mode 100644 index 17542f4d0e..0000000000 --- a/source/mongodb/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys -import site -import setuptools - -# See https://github.com/pypa/pip/issues/7953 -site.ENABLE_USER_SITE = "--user" in sys.argv[1:] - -setuptools.setup() diff --git a/source/mongodb/tests/__init__.py b/source/mongodb/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/mongodb/tests/test_source.py b/source/mongodb/tests/test_source.py deleted file mode 100644 index c39f17f395..0000000000 --- a/source/mongodb/tests/test_source.py +++ /dev/null @@ -1,56 +0,0 @@ -import socket -import inspect -import contextlib -from unittest.mock import patch - -from dffml.util.testing.source import SourceTest -from dffml.util.asynctestcase import AsyncTestCase - -from dffml_source_mongodb.source import MongoDBSourceConfig, MongoDBSource - -from dffml_source_mongodb.util.mongodb_docker import ( - mongodb, - DOCKER_ENV, - DEFAULT_PORT, -) - - -class TestMongoDBSource(AsyncTestCase, SourceTest): - - JS_SETUP = """""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls._exit_stack = contextlib.ExitStack() - cls.exit_stack = cls._exit_stack.__enter__() - cls.container_ip = cls.exit_stack.enter_context(mongodb()) - cls.source_config = MongoDBSourceConfig( - uri=f'mongodb://{DOCKER_ENV["MONGO_INITDB_ROOT_USERNAME"]}:{DOCKER_ENV["MONGO_INITDB_ROOT_PASSWORD"]}@mongodb.unittest:{DEFAULT_PORT}/', - db="mydb", - collection="mycollection", - ) - # Make it so that when the client tries to connect to mongodb.unittest the - # address it gets back is the one for the container - cls.exit_stack.enter_context( - patch( - "socket.getaddrinfo", - return_value=[ - ( - socket.AF_INET, - socket.SOCK_STREAM, - 6, - "", - (cls.container_ip, DEFAULT_PORT), - ) - ], - ) - ) - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - cls._exit_stack.__exit__(None, None, None) - - async def setUpSource(self): - return MongoDBSource(self.source_config)