Merge pull request #25 from alercebroker/fix/models

fix(models): add oid field to detection and non detection

.github/workflows/black.yml

@@ -0,0 +1,13 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "db_plugins/"

.github/workflows/unittests.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python dependencies and run tests
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: UnitTests
@@ -21,10 +21,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install black pytest coverage alchemy_mock mongomock
+        pip install pytest coverage alchemy_mock mongomock
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Black Code Formatter
-      uses: lgeiger/[email protected]
     - name: Test with pytest
       run: |
         coverage run --source db_plugins --omit db_plugins/db/sql/serializers.py -m pytest tests/unittest/

db_plugins/cli/manage.py

@@ -29,15 +29,11 @@ def initdb(settings_path, db=None):
     del sys.modules["settings"]
     if "SQL" in DB_CONFIG:
         init_sql(DB_CONFIG["SQL"], db)
-        click.echo(
-            "Database created with credentials from {}".format(settings_path)
-        )
+        click.echo("Database created with credentials from {}".format(settings_path))
 
     elif "MONGO" in DB_CONFIG:
         init_mongo(DB_CONFIG["MONGO"], db)
-        click.echo(
-            "Database created with credentials from {}".format(settings_path)
-        )
+        click.echo("Database created with credentials from {}".format(settings_path))
 
     else:
         raise Exception("Invalid settings file")
@@ -73,9 +69,7 @@ def migrate(settings_path):
     sys.path.pop(-1)
     if "SQL" in DB_CONFIG:
         migrate_sql()
-        click.echo(
-            "Migrated database with config from {}".format(settings_path)
-        )
+        click.echo("Migrated database with config from {}".format(settings_path))
 
     else:
         print("ERROR", DB_CONFIG)

db_plugins/db/mongo/connection.py

@@ -54,7 +54,7 @@ def connect(self, config):
             username=config["USER"],
             password=config["PASSWORD"],
             port=config["PORT"],
-            authSource=config["DATABASE"]
+            authSource=config["DATABASE"],
         )
         self.base.set_database(config["DATABASE"])
         self.database = self.client[config["DATABASE"]]

db_plugins/db/mongo/models.py

@@ -64,6 +64,7 @@ class Detection(Base, generic_models.Detection):
         Field()
     )  # Telescope id (this gives the spatial coordinates of the observatory, e.g. ZTF, ATLAS-HKO, ATLAS-MLO)
     aid = Field()
+    oid = Field()
     candid = Field()
     mjd = Field()
     fid = Field()
@@ -90,6 +91,7 @@ class NonDetection(Base, generic_models.NonDetection):
 
     aid = Field()
     tid = Field()
+    oid = Field()
     mjd = Field()
     diffmaglim = Field()
     fid = Field()

db_plugins/db/mongo/query.py

@@ -136,9 +136,7 @@ def update(self, instance, args):
 
 
 def bulk_update_creator(collection_class):
-    def bulk_update(
-        self, instances: list, args: list, filter_fields: list = []
-    ):
+    def bulk_update(self, instances: list, args: list, filter_fields: list = []):
         model = type(instances[0])
         self.init_collection(model)
         requests = []
@@ -149,9 +147,7 @@ def bulk_update(
                     {"$set": args[i]},
                 )
             )
-        return collection_class.bulk_write(
-            self, requests=requests, ordered=False
-        )
+        return collection_class.bulk_write(self, requests=requests, ordered=False)
 
     return bulk_update
 
@@ -242,9 +238,7 @@ def find_one(self, filter_by={}, model: Base = None):
 
 
 def find_all_creator(collection_class):
-    def find_all(
-        self, model: Base = None, filter_by={}, paginate=True, **kwargs
-    ):
+    def find_all(self, model: Base = None, filter_by={}, paginate=True, **kwargs):
         """Find list of items of the specified model.
 
         If there are too many items a timeout can happen.

db_plugins/db/sql/migrations/env.py

@@ -21,15 +21,15 @@
 
 db_credentials = db_config["SQLALCHEMY_DATABASE_URL"]
 
-config.set_main_option('sqlalchemy.url', db_credentials)
+config.set_main_option("sqlalchemy.url", db_credentials)
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 target_metadata = Base.metadata
-#target_metadata = None
+# target_metadata = None
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -75,9 +75,7 @@ def run_migrations_online():
     )
 
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()

tests/integration/integration_test.py

@@ -9,9 +9,6 @@
 from db_plugins.db.generic import Pagination
 from sqlalchemy.engine.reflection import Inspector
 import unittest
-import json
-import time
-import datetime
 
 
 class SQLConnectionTest(unittest.TestCase):

tests/unittest/cli/test_manage.py

@@ -21,9 +21,7 @@ def test_init_sql(self, main_mock, mock_makedirs, mock_connection):
 
     @mock.patch("db_plugins.cli.manage.init_sql")
     def test_initdb_sql(self, mock_init_sql):
-        with self.runner.isolated_filesystem(
-            temp_dir=self.settings_path
-        ) as td:
+        with self.runner.isolated_filesystem(temp_dir=self.settings_path) as td:
             with open("settings.py", "w") as f:
                 f.write("DB_CONFIG=")
                 DB_CONFIG = {
@@ -39,8 +37,7 @@ def test_initdb_sql(self, mock_init_sql):
             assert result.exit_code == 0
             mock_init_sql.assert_called()
             assert (
-                "Database created with credentials from {}".format(td)
-                in result.output
+                "Database created with credentials from {}".format(td) in result.output
             )
 
     def test_initdb_error(self):
@@ -50,42 +47,32 @@ def test_initdb_error(self):
 
     @mock.patch("db_plugins.db.sql.initialization.alembic.config.main")
     def test_make_migrations(self, main_mock):
-        with self.runner.isolated_filesystem(
-            temp_dir=self.settings_path
-        ) as td:
+        with self.runner.isolated_filesystem(temp_dir=self.settings_path) as td:
             with open("settings.py", "w") as f:
                 f.write("DB_CONFIG=")
                 DB_CONFIG = {
                     "SQL": {"SQLALCHEMY_DATABASE_URL": "sqlite:///:memory:"},
                 }
                 f.write(str(DB_CONFIG))
-            result = self.runner.invoke(
-                manage.make_migrations, ["--settings_path", td]
-            )
+            result = self.runner.invoke(manage.make_migrations, ["--settings_path", td])
             assert result.exit_code == 0
             main_mock.assert_called()
 
     def test_make_migrations_error(self):
-        result = self.runner.invoke(
-            manage.make_migrations, "--settings_path fail"
-        )
+        result = self.runner.invoke(manage.make_migrations, "--settings_path fail")
         assert result.exit_code != 0
         assert "Settings file not found" == str(result.exception)
 
     @mock.patch("db_plugins.db.sql.initialization.alembic.config.main")
     def test_migrate(self, main_mock):
-        with self.runner.isolated_filesystem(
-            temp_dir=self.settings_path
-        ) as td:
+        with self.runner.isolated_filesystem(temp_dir=self.settings_path) as td:
             with open("settings.py", "w") as f:
                 f.write("DB_CONFIG=")
                 DB_CONFIG = {
                     "SQL": {"SQLALCHEMY_DATABASE_URL": "sqlite:///:memory:"},
                 }
                 f.write(str(DB_CONFIG))
-            result = self.runner.invoke(
-                manage.migrate, ["--settings_path", td]
-            )
+            result = self.runner.invoke(manage.migrate, ["--settings_path", td])
             assert result.exit_code == 0
             main_mock.assert_called()
 
@@ -110,9 +97,7 @@ def test_init_mongo(self, mock_connection):
 
     @mock.patch("db_plugins.cli.manage.init_mongo")
     def test_initdb_mongo(self, mock_init_mongo):
-        with self.runner.isolated_filesystem(
-            temp_dir=self.settings_path
-        ) as td:
+        with self.runner.isolated_filesystem(temp_dir=self.settings_path) as td:
             with open("settings.py", "w") as f:
                 f.write("DB_CONFIG=")
                 DB_CONFIG = {
@@ -129,6 +114,5 @@ def test_initdb_mongo(self, mock_init_mongo):
             assert result.exit_code == 0
             mock_init_mongo.assert_called()
             assert (
-                "Database created with credentials from {}".format(td)
-                in result.output
+                "Database created with credentials from {}".format(td) in result.output
             )

tests/unittest/db/test_mongo.py

@@ -37,9 +37,7 @@ def test_connect(self):
 
     def test_create_db(self):
         self.conn.create_db()
-        collections = self.client[
-            self.config["DATABASE"]
-        ].list_collection_names()
+        collections = self.client[self.config["DATABASE"]].list_collection_names()
         expected = ["object", "detection", "non_detection"]
         self.assertEqual(collections, expected)
 
@@ -94,9 +92,7 @@ def setUp(self):
         self.database = client["database"]
         self.obj_collection = self.database["object"]
         self.obj_collection.insert_one({"test": "test"})
-        self.mongo_query_class = mongo_query_creator(
-            mongomock.collection.Collection
-        )
+        self.mongo_query_class = mongo_query_creator(mongomock.collection.Collection)
         self.query = self.mongo_query_class(
             model=Object,
             database=self.database,