Merge pull request IntegriChain1#37 from norton120/update-session-man…

…ager

Makes Session Manager Work for UAT / Prod

config/core_project.yaml

@@ -13,4 +13,4 @@ AWS_ACCOUNT: "265991248033"
 AWS_REGION: us-east-1
 AWS_BATCH_TEST_JOB_QUEUE: core
 LOGGING_CONFIG: logging.yaml
-
+DEV_CONFIGURATION_APPLICATION_CONN_STRING: "postgresql://configurator:configurator@configurationpg/configuration_application"

core/database/env.py

@@ -5,7 +5,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool
-from core.models.configuration import Base
+from core.models.configuration import Base, GenerateEngine
 from core.secret import Secret
 from core.constants import ENVIRONMENT
 from alembic import context
@@ -29,25 +29,6 @@
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
-def create_conn_string_from_secret():
-    """ builds the appropriate db string based on ENV - specific secret.
-        For dev uses the value in alembic.ini."""
-    if ENVIRONMENT == "dev":
-        return config.get_main_option("sqlalchemy.url")
-
-    secret = Secret(env=ENVIRONMENT, 
-                    name='configuration_application',
-                    type_of='database',
-                    mode='read'
-                    )
-    if secret.rdbms == "postgres":
-        conn_string = f"postgresql://{secret.user}:{secret.password}@{host}/{database}"
-    else:
-        m = "Only postgres databases are supported for configuration_application at this time."
-        logger.critical(m)
-        raise NotImplementedError(m)
-    return conn_string
-
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -60,7 +41,7 @@ def run_migrations_offline():
     script output.
 
     """
-    url = create_conn_string_from_secret()    
+    url = GenerateEngine().url    
     context.configure(
         url=url, target_metadata=target_metadata, literal_binds=True
     )
@@ -76,8 +57,7 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    connectable = create_engine(create_conn_string_from_secret())
-
+    connectable = GenerateEngine().get_engine()
     with connectable.connect() as connection:
         context.configure(
             connection=connection, target_metadata=target_metadata

core/helpers/configuration_mocker.py

@@ -10,9 +10,7 @@ class ConfigurationMocker(LoggerMixin):
     '''
 
     def __init__(self)-> None:
-        # TODO: migrate to local postgres instance
-        engine = create_engine('sqlite://')
-        #engine = config.GenerateEngine(env='dev', local=True).get_engine()
+        engine = config.GenerateEngine().get_engine()
 
         # this instansiates the in-memory sqlite instance
         config.Base.metadata.create_all(engine)
@@ -82,7 +80,8 @@ def _mock_pipelines(self)-> None:
               pipeline_type_id=1, run_frequency='daily'),
             p(id=2, name="bluth_profitability", brand_id=2,
               pipeline_type_id=2, run_frequency='hourly'),
-            p(id=3, name="temocil_profitablility", brand_id=1, pipeline_type_id=1, run_frequency='daily'),
+            p(id=3, name="temocil_profitablility", brand_id=1,
+              pipeline_type_id=1, run_frequency='daily'),
             p(id=500, name="bluth_banana_regression_deprecated", brand_id=2,
               pipeline_type_id=1, is_active=False, run_frequency='hourly')])
         self.session.commit()

core/helpers/docker.py

@@ -11,7 +11,7 @@
 # commit hash, the work-around is to use the BRANCH_NAME env var that
 # Jenkins sets.
 def get_branch_name():
-    repo = Repo('.')
+    repo = Repo(ProjectRoot().get_path())
     try:
         return repo.active_branch.name
     except:

core/helpers/session_helper.py

@@ -1,24 +1,33 @@
 from core.constants import ENVIRONMENT
 from core.helpers.configuration_mocker import ConfigurationMocker as CMock
+import core.models.configuration as config
 from sqlalchemy.orm.session import Session
+from core.logging import LoggerMixin
 
-class SessionHelper:
+class SessionHelper(LoggerMixin):
     """ Gets the correct env-based configuration secret,
         returns a session to the right configuration db. 
+        For the dev env it pre-populates the database with helper seed data.
     """
+
     def __init__(self):
         self._session = None
+        self.logger.info(f"Creating session for {ENVIRONMENT} environment...")
         if ENVIRONMENT == "dev":
             cmock = CMock()
             cmock.generate_mocks()
             self._session = cmock.get_session()
-        elif ENVIRONMENT == "prod":
-            pass
+            self.logger.info("Done. Created dev session with mock data.")    
+        if ENVIRONMENT in ("prod", "uat"):
+            engine = config.GenerateEngine().get_engine()
+            session = config.Session(engine)
+            self._session = session.get_session()
+            self.logger.info(f"Done. Created {ENVIRONMENT} session.")
 
     @property
     def session(self)-> Session:
         return self._session
 
     @session.setter
-    def session(self,session)->None:
+    def session(self, session)->None:
         raise ValueError("session cannot be explicitly set in session_helper.")

core/models/configuration.py

@@ -1,7 +1,8 @@
 from sqlalchemy import engine, create_engine, Column, Integer, String, Boolean, TIMESTAMP, text, ForeignKey, func
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import session, sessionmaker, relationship
-
+from core.constants import DEV_CONFIGURATION_APPLICATION_CONN_STRING, ENVIRONMENT
+from core.secret import Secret
 Base = declarative_base()
 
 
@@ -18,25 +19,45 @@ def __init__(self, engine: engine.base.Engine) -> None:
     def get_session(self) -> session.Session:
         return self.session
 
-
 class GenerateEngine:
-    """ abstract defining connections here. Local assumes a psql instance on the metal. """
-
-    def __init__(self, env: str, local: bool = False) -> None:
-        if local:
-            self.engine = self._local_engine()
+    """ abstract defining connections here. Local assumes a psql instance in a local docker container. """
+
+    def __init__(self, in_memory:bool=True) -> None:
+        """ So the default development configuration database is an in-memory sqlite instance. 
+            This is fast and easy and atomic (resets after every execution) but it is NOT exactly the same as 
+            the prod PG instance. When we want an identical configuration environment, setting in_memory to False 
+            gives you the local docker container PG. """
+
+        if ENVIRONMENT == "dev":
+            if in_memory:
+                self._url = "sqlite://"
+            else:
+                self._url = DEV_CONFIGURATION_APPLICATION_CONN_STRING
         else:
-            self.engine = self._secret_defined_engine()
+            self._url = self._secret_defined_url()
 
-    def get_engine(self) -> engine.base.Engine:
-        return self.engine
+    @property
+    def url(self) -> str: 
+        return self._url
 
-    def _local_engine(self) -> engine.base.Engine:
-        return create_engine('postgresql://configurator:configurator@localhost/configuration_application')
-
-    def _secret_defined_engine(self) -> engine.base.Engine:
-        # TODO: in DC-57 update this to use secret
-        pass
+    def get_engine(self) -> engine.base.Engine:
+        engine = create_engine(self._url)
+        return engine
+
+    def _secret_defined_url(self) -> str:
+        """ creates a session connecting to the correct configuration_application db based on ENV."""
+        secret = Secret(env=ENVIRONMENT, 
+                        name='configuration_application',
+                        type_of='database',
+                        mode='read'
+                        )
+        if secret.rdbms == "postgres":
+            conn_string = f"postgresql://{secret.user}:{secret.password}@{host}/{database}"
+        else:
+            m = "Only postgres databases are supported for configuration_application at this time."
+            logger.critical(m)
+            raise NotImplementedError(m)
+        return conn_string
 
 
 """Mixins

tests/unit/test_docker.py

@@ -1,10 +1,16 @@
 import pytest
 from core.helpers import docker
 from git import Repo
+from core.helpers.project_root import ProjectRoot
+import os
 
 class Test():
     def setup(self):
-        self.branch_name = Repo('.').active_branch.name
+        repo = Repo(ProjectRoot().get_path())
+        try:
+            self.branch_name =  repo.active_branch.name
+        except:
+            self.branch_name =  os.environ['BRANCH_NAME']
 
     def test_get_core_tag(self):
         self.setup()

tests/unit/test_helpers.py

@@ -5,7 +5,8 @@
 from core.helpers.project_root import ProjectRoot
 from core.helpers.configuration_mocker import ConfigurationMocker as CMock
 import core.models.configuration as config
-from core.helpers.s3_naming_helper import S3NamingHelper
+from core.helpers.session_helper import SessionHelper
+from core.helpers.s3_naming_helper import S3NamingHelper 
 from core.helpers.file_mover import FileMover, FileDestination
 from core.helpers.session_helper import SessionHelper
 
@@ -178,3 +179,31 @@ def test_get_file_type(paramiko_trans, paramiko_sftp):
     fd = [FileDestination(regex="n^", file_type="none")]
     ft = fm.get_file_type(test_file, fd)
     assert ft, "dont_move"
+
+## SessionHelper
+
+@patch("core.helpers.session_helper.config")
+@patch("core.helpers.session_helper.CMock")
+@patch("core.helpers.session_helper.ENVIRONMENT","prod")
+def test_session_helper_prod(mock_cmock, mock_config):
+    session = SessionHelper().session
+    assert mock_config.GenerateEngine.called
+    assert mock_config.Session.called
+    assert not mock_cmock.called
+
+@patch("core.helpers.session_helper.config")
+@patch("core.helpers.session_helper.CMock")
+@patch("core.helpers.session_helper.ENVIRONMENT","uat")
+def test_session_helper_uat(mock_cmock, mock_config):
+    session = SessionHelper().session
+    assert mock_config.GenerateEngine.called
+    assert mock_config.Session.called
+    assert not mock_cmock.called
+
+@patch("core.helpers.session_helper.config")
+@patch("core.helpers.session_helper.CMock")
+@patch("core.helpers.session_helper.ENVIRONMENT","dev")
+def test_session_helper_dev(mock_cmock, mock_config):
+    session = SessionHelper().session
+    assert not mock_config.GenerateEngine.called
+    assert mock_cmock.called