diff --git a/config/core_project.yaml b/config/core_project.yaml index e5963ca7..1d4d8a75 100644 --- a/config/core_project.yaml +++ b/config/core_project.yaml @@ -13,4 +13,4 @@ AWS_ACCOUNT: "265991248033" AWS_REGION: us-east-1 AWS_BATCH_TEST_JOB_QUEUE: core LOGGING_CONFIG: logging.yaml - +DEV_CONFIGURATION_APPLICATION_CONN_STRING: "postgresql://configurator:configurator@configurationpg/configuration_application" diff --git a/core/database/env.py b/core/database/env.py index 4f0d9e17..83334ee3 100644 --- a/core/database/env.py +++ b/core/database/env.py @@ -5,7 +5,7 @@ from sqlalchemy import create_engine from sqlalchemy import engine_from_config from sqlalchemy import pool -from core.models.configuration import Base +from core.models.configuration import Base, GenerateEngine from core.secret import Secret from core.constants import ENVIRONMENT from alembic import context @@ -29,25 +29,6 @@ # my_important_option = config.get_main_option("my_important_option") # ... etc. -def create_conn_string_from_secret(): - """ builds the appropriate db string based on ENV - specific secret. - For dev uses the value in alembic.ini.""" - if ENVIRONMENT == "dev": - return config.get_main_option("sqlalchemy.url") - - secret = Secret(env=ENVIRONMENT, - name='configuration_application', - type_of='database', - mode='read' - ) - if secret.rdbms == "postgres": - conn_string = f"postgresql://{secret.user}:{secret.password}@{host}/{database}" - else: - m = "Only postgres databases are supported for configuration_application at this time." - logger.critical(m) - raise NotImplementedError(m) - return conn_string - def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -60,7 +41,7 @@ def run_migrations_offline(): script output. """ - url = create_conn_string_from_secret() + url = GenerateEngine().url context.configure( url=url, target_metadata=target_metadata, literal_binds=True ) @@ -76,8 +57,7 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = create_engine(create_conn_string_from_secret()) - + connectable = GenerateEngine().get_engine() with connectable.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata diff --git a/core/helpers/configuration_mocker.py b/core/helpers/configuration_mocker.py index 1bfa8ae5..514654a4 100644 --- a/core/helpers/configuration_mocker.py +++ b/core/helpers/configuration_mocker.py @@ -10,9 +10,7 @@ class ConfigurationMocker(LoggerMixin): ''' def __init__(self)-> None: - # TODO: migrate to local postgres instance - engine = create_engine('sqlite://') - #engine = config.GenerateEngine(env='dev', local=True).get_engine() + engine = config.GenerateEngine().get_engine() # this instansiates the in-memory sqlite instance config.Base.metadata.create_all(engine) @@ -82,7 +80,8 @@ def _mock_pipelines(self)-> None: pipeline_type_id=1, run_frequency='daily'), p(id=2, name="bluth_profitability", brand_id=2, pipeline_type_id=2, run_frequency='hourly'), - p(id=3, name="temocil_profitablility", brand_id=1, pipeline_type_id=1, run_frequency='daily'), + p(id=3, name="temocil_profitablility", brand_id=1, + pipeline_type_id=1, run_frequency='daily'), p(id=500, name="bluth_banana_regression_deprecated", brand_id=2, pipeline_type_id=1, is_active=False, run_frequency='hourly')]) self.session.commit() diff --git a/core/helpers/docker.py b/core/helpers/docker.py index 22dec2c9..9cccd458 100644 --- a/core/helpers/docker.py +++ b/core/helpers/docker.py @@ -11,7 +11,7 @@ # commit hash, the work-around is to use the BRANCH_NAME env var that # Jenkins sets. def get_branch_name(): - repo = Repo('.') + repo = Repo(ProjectRoot().get_path()) try: return repo.active_branch.name except: diff --git a/core/helpers/session_helper.py b/core/helpers/session_helper.py index 8339187b..0d75154a 100644 --- a/core/helpers/session_helper.py +++ b/core/helpers/session_helper.py @@ -1,24 +1,33 @@ from core.constants import ENVIRONMENT from core.helpers.configuration_mocker import ConfigurationMocker as CMock +import core.models.configuration as config from sqlalchemy.orm.session import Session +from core.logging import LoggerMixin -class SessionHelper: +class SessionHelper(LoggerMixin): """ Gets the correct env-based configuration secret, returns a session to the right configuration db. + For the dev env it pre-populates the database with helper seed data. """ + def __init__(self): self._session = None + self.logger.info(f"Creating session for {ENVIRONMENT} environment...") if ENVIRONMENT == "dev": cmock = CMock() cmock.generate_mocks() self._session = cmock.get_session() - elif ENVIRONMENT == "prod": - pass + self.logger.info("Done. Created dev session with mock data.") + if ENVIRONMENT in ("prod", "uat"): + engine = config.GenerateEngine().get_engine() + session = config.Session(engine) + self._session = session.get_session() + self.logger.info(f"Done. Created {ENVIRONMENT} session.") @property def session(self)-> Session: return self._session @session.setter - def session(self,session)->None: + def session(self, session)->None: raise ValueError("session cannot be explicitly set in session_helper.") diff --git a/core/models/configuration.py b/core/models/configuration.py index 9f609956..ff8b3ee4 100644 --- a/core/models/configuration.py +++ b/core/models/configuration.py @@ -1,7 +1,8 @@ from sqlalchemy import engine, create_engine, Column, Integer, String, Boolean, TIMESTAMP, text, ForeignKey, func from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import session, sessionmaker, relationship - +from core.constants import DEV_CONFIGURATION_APPLICATION_CONN_STRING, ENVIRONMENT +from core.secret import Secret Base = declarative_base() @@ -18,25 +19,45 @@ def __init__(self, engine: engine.base.Engine) -> None: def get_session(self) -> session.Session: return self.session - class GenerateEngine: - """ abstract defining connections here. Local assumes a psql instance on the metal. """ - - def __init__(self, env: str, local: bool = False) -> None: - if local: - self.engine = self._local_engine() + """ abstract defining connections here. Local assumes a psql instance in a local docker container. """ + + def __init__(self, in_memory:bool=True) -> None: + """ So the default development configuration database is an in-memory sqlite instance. + This is fast and easy and atomic (resets after every execution) but it is NOT exactly the same as + the prod PG instance. When we want an identical configuration environment, setting in_memory to False + gives you the local docker container PG. """ + + if ENVIRONMENT == "dev": + if in_memory: + self._url = "sqlite://" + else: + self._url = DEV_CONFIGURATION_APPLICATION_CONN_STRING else: - self.engine = self._secret_defined_engine() + self._url = self._secret_defined_url() - def get_engine(self) -> engine.base.Engine: - return self.engine + @property + def url(self) -> str: + return self._url - def _local_engine(self) -> engine.base.Engine: - return create_engine('postgresql://configurator:configurator@localhost/configuration_application') - - def _secret_defined_engine(self) -> engine.base.Engine: - # TODO: in DC-57 update this to use secret - pass + def get_engine(self) -> engine.base.Engine: + engine = create_engine(self._url) + return engine + + def _secret_defined_url(self) -> str: + """ creates a session connecting to the correct configuration_application db based on ENV.""" + secret = Secret(env=ENVIRONMENT, + name='configuration_application', + type_of='database', + mode='read' + ) + if secret.rdbms == "postgres": + conn_string = f"postgresql://{secret.user}:{secret.password}@{host}/{database}" + else: + m = "Only postgres databases are supported for configuration_application at this time." + logger.critical(m) + raise NotImplementedError(m) + return conn_string """Mixins diff --git a/tests/unit/test_docker.py b/tests/unit/test_docker.py index 9af944a3..e4545b61 100644 --- a/tests/unit/test_docker.py +++ b/tests/unit/test_docker.py @@ -1,10 +1,16 @@ import pytest from core.helpers import docker from git import Repo +from core.helpers.project_root import ProjectRoot +import os class Test(): def setup(self): - self.branch_name = Repo('.').active_branch.name + repo = Repo(ProjectRoot().get_path()) + try: + self.branch_name = repo.active_branch.name + except: + self.branch_name = os.environ['BRANCH_NAME'] def test_get_core_tag(self): self.setup() diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index ecd2a683..df2d9d49 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -5,7 +5,8 @@ from core.helpers.project_root import ProjectRoot from core.helpers.configuration_mocker import ConfigurationMocker as CMock import core.models.configuration as config -from core.helpers.s3_naming_helper import S3NamingHelper +from core.helpers.session_helper import SessionHelper +from core.helpers.s3_naming_helper import S3NamingHelper from core.helpers.file_mover import FileMover, FileDestination from core.helpers.session_helper import SessionHelper @@ -178,3 +179,31 @@ def test_get_file_type(paramiko_trans, paramiko_sftp): fd = [FileDestination(regex="n^", file_type="none")] ft = fm.get_file_type(test_file, fd) assert ft, "dont_move" + +## SessionHelper + +@patch("core.helpers.session_helper.config") +@patch("core.helpers.session_helper.CMock") +@patch("core.helpers.session_helper.ENVIRONMENT","prod") +def test_session_helper_prod(mock_cmock, mock_config): + session = SessionHelper().session + assert mock_config.GenerateEngine.called + assert mock_config.Session.called + assert not mock_cmock.called + +@patch("core.helpers.session_helper.config") +@patch("core.helpers.session_helper.CMock") +@patch("core.helpers.session_helper.ENVIRONMENT","uat") +def test_session_helper_uat(mock_cmock, mock_config): + session = SessionHelper().session + assert mock_config.GenerateEngine.called + assert mock_config.Session.called + assert not mock_cmock.called + +@patch("core.helpers.session_helper.config") +@patch("core.helpers.session_helper.CMock") +@patch("core.helpers.session_helper.ENVIRONMENT","dev") +def test_session_helper_dev(mock_cmock, mock_config): + session = SessionHelper().session + assert not mock_config.GenerateEngine.called + assert mock_cmock.called