From 95e75a13674e5c6e963b1f8ac7097bfa40b05025 Mon Sep 17 00:00:00 2001 From: "Rayne N. Schaffer" Date: Mon, 28 Jan 2019 12:46:31 -0500 Subject: [PATCH 1/9] extract transform first pass through with config mocks --- core/cli.py | 13 +++++ core/contract.py | 20 ++++++- core/helpers/configuration_mocker.py | 18 +++--- core/helpers/file_mover.py | 69 +++++++++++++++++++++++ requirements.txt | 1 + tests/unit/test_helpers.py | 38 ++++++++++++- transforms/shared/raw/extract.py | 82 ++++++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 11 deletions(-) create mode 100644 core/helpers/file_mover.py create mode 100644 transforms/shared/raw/extract.py diff --git a/core/cli.py b/core/cli.py index 22744bdc..f22de2cb 100644 --- a/core/cli.py +++ b/core/cli.py @@ -4,6 +4,8 @@ DOCKER_REPO = 'ichain/core' +from transforms.shared.raw import extract + @click.group() def cli(): # pragma: no cover pass @@ -15,6 +17,17 @@ def add(a, b): click.echo(print(a + b)) return a + b +@cli.command() +@click.argument('env',type=click.Choice(['dev'])) +@click.argument('manufacturer', type=str) +@click.argument('brand', type=str) +@click.argument('id', type=int) +def run_extract(env,manufacturer,brand,id): + repo = Repo('.') + branch_name = repo.active_branch.name + extract.test_run_extract_transform(env=env,transform_id=id, branch=branch_name, manufacturer=manufacturer, brand=brand) + + @cli.command() @click.argument('env', type=click.Choice(['local'])) def publish(env): diff --git a/core/contract.py b/core/contract.py index 5a143a7b..eceb8725 100644 --- a/core/contract.py +++ b/core/contract.py @@ -3,6 +3,9 @@ import logging from git import Repo from core.helpers.s3_naming_helper import S3NamingHelper as s3Name +import boto3 +from botocore.errorfactory import ClientError +import botocore class Contract: @@ -28,7 +31,7 @@ class Contract: SUB-PARTITION - for datasets, the sub-partitions add additional partitioning with additional prefixes FILENAME - nondescript in the contract ''' - DEV = 'ichain-development' + DEV = 'ichain-dev-gluepoc' PROD = 'ichain-production' UAT = 'ichain-uat' STATES = ['raw', 'ingest', 'master', 'enhance', @@ -229,6 +232,21 @@ def publish_raw_file(self, local_file_path: str) ->None: s3_client.upload_fileobj(file_data, Bucket=self.get_bucket( ), Key=self.get_key(), ExtraArgs={"Metadata": extra_args}) + def get_raw_file_metadata(self, local_file_path:str) ->None: + # If file exists, return its metadata + s3_client = boto3.client('s3') + + self.set_file_name(os.path.split(local_file_path)[1]) + try: + return s3_client.head_object(Bucket=self.get_bucket(),Key=self.get_key()) + except ClientError as e: + # If file does not exist, throw back since it needs to be moved anyways + # Consider: cleaner handling? + if e.response['ResponseMetadata']['HTTPStatusCode'] == 404: + raise e + else: + raise e + # aliases def get_brand(self)->str: diff --git a/core/helpers/configuration_mocker.py b/core/helpers/configuration_mocker.py index d0e6e7cf..5acce72b 100644 --- a/core/helpers/configuration_mocker.py +++ b/core/helpers/configuration_mocker.py @@ -50,17 +50,17 @@ def _mock_extract_configurations(self)-> None: ex = config.ExtractConfiguration self.session.add_all([ ex(id=1, transformation_id=2, filesystem_path='', - prefix='', secret_name='bluth'), + prefix='', secret_name='dev-sftp'), ex(id=2, transformation_id=2, filesystem_path='banana_stand_data', - prefix='gob', secret_name='bluth'), + prefix='gob', secret_name='dev-sftp'), ex(id=3, transformation_id=3, filesystem_path='sudden_valley_holdings', - prefix='', secret_name='bluth'), - ex(id=4, transformation_id=1, filesystem_path='', - prefix='', secret_name='sitwell'), - ex(id=5, transformation_id=1, filesystem_path='', - prefix='001545', secret_name='sitwell'), - ex(id=6, transformation_id=1, filesystem_path='200-1', - prefix='', secret_name='sitwell') + prefix='', secret_name='dev-sftp'), + ex(id=4, transformation_id=1, filesystem_path='/incoming', + prefix='', secret_name='dev-sftp'), + ex(id=5, transformation_id=1, filesystem_path='/incoming', + prefix='test-extract-root-prefix', secret_name='dev-sftp'), + ex(id=6, transformation_id=1, filesystem_path='/incoming/testing_extract', + prefix='', secret_name='dev-sftp') ]) self.session.commit() logging.debug('Done generating extract_configuration mocks.') diff --git a/core/helpers/file_mover.py b/core/helpers/file_mover.py new file mode 100644 index 00000000..94968a1e --- /dev/null +++ b/core/helpers/file_mover.py @@ -0,0 +1,69 @@ +import paramiko +import stat +import os +from typing import NamedTuple +import re + +class FileDestination(NamedTuple): + regex: str + file_type: str + + +class FileMover(): + def __init__(self,secret): + user = secret.user + password = secret.password + host = secret.host + port = secret.port + mode = secret.mode + self.transport = paramiko.Transport(host, port) + self.transport.connect(username=user, password=password) + self.sftp = paramiko.SFTPClient.from_transport(self.transport) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.sftp.close() + self.transport.close() + + def get_file(self, remote_path: str, local_path: str): + # Fetch file from remote + # Set local file time to match remote for comparison to S3 modified time + utime = self.sftp.stat(remote_path).st_mtime + self.sftp.get(remote_path, local_path) + os.utime(local_path, (utime,utime)) + + def get_file_type(self, filename, file_dest_map): + # Check if file is matching to the prefix, otherwise don't move + file_type = [x.file_type for x in file_dest_map if re.match(x.regex, filename)] + if len(file_type) < 1: + return 'dont_move' + else: + return file_type[0] + + def is_dir(self, remote_file) -> bool: + # Return bool of if 'file' is a directory or not + return stat.S_ISDIR(remote_file.st_mode) + + def list_files(self, sftp_prefix: str): + # List all files on remote + return self.sftp.listdir_attr(sftp_prefix) + + +def get_files(tmp_dir:str,prefix: str, remote_path: str, secret): + # Set file filtering + files_dest = [FileDestination(f"^{prefix}.*$","do_move")] + + # Open SFTP connection + with FileMover(secret=secret) as fm: + file_list = fm.list_files(remote_path) + for remote_file in file_list: + # For every "file" in the remote list, make sure its not a directory and matches filters + if not (fm.is_dir(remote_file)) and (fm.get_file_type(remote_file.filename, files_dest)!='dont_move'): + remote_file_path = remote_path + "/" + remote_file.filename + # Set file name to include the path, in case of duplicate file names in different locations + local_file_name = remote_file_path.replace("/",".")[1:] + local_file_path = os.path.join(tmp_dir, local_file_name) + + fm.get_file(remote_file_path, local_file_path) diff --git a/requirements.txt b/requirements.txt index bb72d76f..992ca3cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ psycopg2==2.7.6.1 alembic==1.0.6 papermill==0.17.1 jupyter==1.0.0 +paramiko diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index afa0a837..def8f937 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -1,10 +1,12 @@ import moto import pytest +from unittest.mock import patch import os from core.helpers.project_root import ProjectRoot from core.helpers.configuration_mocker import ConfigurationMocker as CMock import core.models.configuration as config from core.helpers.s3_naming_helper import S3NamingHelper +from core.helpers.file_mover import FileMover, FileDestination def test_project_root_in_project(): root = ProjectRoot() @@ -44,7 +46,7 @@ def test_mock_transformation_relationships(): assert len(secrets) == 3 - assert set(secrets) == set(['sitwell']) + assert set(secrets) == set(['dev-sftp']) ## S3 Naming Helper @@ -114,3 +116,37 @@ def test_validate_s3_path(): response = helper.validate_s3_path('s3://bucket/path/all/good') assert response[0], f'disallowed good s3 path' + + +class secret_mock(): + def __init__(self): + self.user = 'test_user' + self.password = 'test_password' + self.host = 'test_host' + self.port = 'test_port' + self.mode = 'test_mode' + +@patch('paramiko.Transport') +@patch('paramiko.SFTPClient.from_transport') +def test_filemover_paramiko(paramiko_trans,paramiko_sftp): + sm = secret_mock() + + with FileMover(sm) as fm: + assert paramiko_trans.called + assert paramiko_sftp.called + +@patch('paramiko.Transport') +@patch('paramiko.SFTPClient') +def test_get_file_type(paramiko_trans,paramiko_sftp): + sm = secret_mock() + fm = FileMover(sm) + test_file = "test_file_name" + + fd = [FileDestination(regex=".*", file_type="all")] + ft = fm.get_file_type(test_file, fd) + assert ft, "all" + + fd = [FileDestination(regex="n^", file_type="none")] + ft = fm.get_file_type(test_file, fd) + assert ft, "dont_move" + diff --git a/transforms/shared/raw/extract.py b/transforms/shared/raw/extract.py new file mode 100644 index 00000000..ea1a4a97 --- /dev/null +++ b/transforms/shared/raw/extract.py @@ -0,0 +1,82 @@ +from core.helpers import configuration_mocker, file_mover +from core.models import configuration +from core import secret, contract + +import os +import tempfile + +def test_run_extract_transform(env:str,transform_id:int,branch:str,manufacturer:str,brand:str): + # TODO: This is currently using the config mocker to generate configs to run against + # When configs are ready, switch out to appropriate functions + + # Mocking configs for use in extract testing + config_mock = configuration_mocker.ConfigurationMocker() + config_mock.generate_mocks() + + session = config_mock.get_session() + + ec = configuration.ExtractConfiguration + t = configuration.Transformation + + # Start querying the extract configs + q = session.query(t).filter(t.id == transform_id) + for f_transform in q: + transform_state = str.lower(f_transform.pipeline_state.pipeline_state_type.name) + for extract in f_transform.extract_configurations: + output_contract = contract.Contract(env=env, + state=transform_state, + branch=branch, + parent=manufacturer, + child=brand + ) + extract_transform = ExtractTransform(env=env, mock_config=extract,output_contract=output_contract) + extract_transform.test_run() + + +class ExtractTransform(): + def __init__(self, env:str, mock_config, output_contract): + self.env = env + self.config=mock_config + self.output_contract = output_contract + + def test_run(self): + # Set values from extract config + remote_path = self.config.filesystem_path + prefix = self.config.prefix + secret_name = self.config.secret_name + + # Fetch secret from secret contract + # TODO: Currently configs made for FTP only, FTP type passed in directly + source_secret = secret.Secret(name=self.config.secret_name,env=self.env,type_of="FTP",mode="write") + + # Get files from remote and start pushing to s3 + with tempfile.TemporaryDirectory() as tmp_dir: + file_mover.get_files(tmp_dir=tmp_dir,prefix=prefix,remote_path=remote_path,secret=source_secret) + self.push_to_s3(tmp_dir) + + + def push_to_s3(self, tmp_dir): + # Get the transformation's output contract + oc = self.output_contract + + # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints + for local_file in os.listdir(f"{tmp_dir}"): + local_file_path = os.path.join(tmp_dir,local_file) + local_file_modified_time = os.stat(os.path.join(tmp_dir,local_file)).st_mtime + + if (self._file_needs_update(local_file_path,local_file_modified_time)): + self.output_contract.publish_raw_file(local_file_path) + + def _file_needs_update(self,local_file_path,local_file_modified_time): + # Check if file needs to be pushed + # File is only considered to need to be pushed if it does not exist or has been modified since last push + output_contract = self.output_contract + + try: + s3_last_modified = output_contract.get_raw_file_metadata(local_file_path)['Metadata']['source_modified_time'] + if (float(s3_last_modified) < float(local_file_modified_time)): + return True + else: + return False + except: + return True From 3f610abac2a461590d5ecfca88300ad03d9db464 Mon Sep 17 00:00:00 2001 From: Nate Sanford Date: Mon, 28 Jan 2019 16:58:03 -0500 Subject: [PATCH 2/9] pair session - move extract into notebook and add extract library --- core/helpers/notebook.py | 26 +++++++++++ core/secret.py | 3 +- core/transforms/__init__.py | 0 core/transforms/shared/__init__.py | 0 core/transforms/shared/raw/__init__.py | 0 core/transforms/shared/raw/extract.py | 55 +++++++++++++++++++++++ core_project.yaml | 2 +- script/notebook | 1 + setup.py | 2 +- transforms/shared/raw/extract.ipynb | 61 ++++++++++++++++++++++++-- 10 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 core/transforms/__init__.py create mode 100644 core/transforms/shared/__init__.py create mode 100644 core/transforms/shared/raw/__init__.py create mode 100644 core/transforms/shared/raw/extract.py diff --git a/core/helpers/notebook.py b/core/helpers/notebook.py index 201323e3..a321751e 100644 --- a/core/helpers/notebook.py +++ b/core/helpers/notebook.py @@ -1,6 +1,10 @@ import papermill as pm from core.helpers import project_root from core.constants import ENV_BUCKET +from core.helpers import configuration_mocker +from core.models import configuration +from core import contract + root = project_root.ProjectRoot() def run_transform(env: str, id: int, input_contract: str, output_contract: str) -> str: @@ -29,3 +33,25 @@ def output_url(output_path: str) -> str: s3_prefix = "s3://{ENV_BUCKET}/notebooks" url_prefix = "http://notebook.integrichain.net/view" return output_path.replace(s3_prefix, url_prefix) + +def get_contract(env, state, branch, parent, child): + kontract = contract.Contract(env=env, + state=state, + branch=branch, + parent=parent, + child=child + ) + return kontract + +def get_transform(transform_id): + config_mock = configuration_mocker.ConfigurationMocker() + config_mock.generate_mocks() + + session = config_mock.get_session() + + ec = configuration.ExtractConfiguration + t = configuration.Transformation + + # Start querying the extract configs + transform = session.query(t).filter(t.id == transform_id).one() + return transform \ No newline at end of file diff --git a/core/secret.py b/core/secret.py index b639d004..533bcae1 100644 --- a/core/secret.py +++ b/core/secret.py @@ -1,6 +1,6 @@ import boto3 import json - +import logging class Secret: ''' Abstracts aws secretsmanager - values of the secret are callable attributes. ie: @@ -53,6 +53,7 @@ def _build_identifier(self, env: str, type_of: str, name: str, mode: str) -> str def _get_secret(self, identifier: str, force_env: bool) -> str: # first look to see if the explicit secret exists + print(f"Secret identifier: {identifier}") try: raw_secret = self.client.get_secret_value(SecretId=identifier) except Exception as e: diff --git a/core/transforms/__init__.py b/core/transforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/transforms/shared/__init__.py b/core/transforms/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/transforms/shared/raw/__init__.py b/core/transforms/shared/raw/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/transforms/shared/raw/extract.py b/core/transforms/shared/raw/extract.py new file mode 100644 index 00000000..6c521f6b --- /dev/null +++ b/core/transforms/shared/raw/extract.py @@ -0,0 +1,55 @@ +from core.helpers import file_mover +from core.models import configuration +from core import secret, contract + +import os +import tempfile + +class ExtractTransform(): + def __init__(self, env:str, transform, output_contract): + self.env = env + self.output_contract = output_contract + self.transform = transform + + def run(self): + for config in self.transform.extract_configurations: + # Set values from extract config + remote_path = config.filesystem_path + prefix = config.prefix + secret_name = config.secret_name + + # Fetch secret from secret contract + # TODO: Currently configs made for FTP only, FTP type passed in directly + source_secret = secret.Secret(name=config.secret_name,env=self.env,type_of="FTP",mode="write") + + # Get files from remote and start pushing to s3 + with tempfile.TemporaryDirectory() as tmp_dir: + file_mover.get_files(tmp_dir=tmp_dir,prefix=prefix,remote_path=remote_path,secret=source_secret) + self.push_to_s3(tmp_dir) + + + def push_to_s3(self, tmp_dir): + # Get the transformation's output contract + oc = self.output_contract + + # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints + for local_file in os.listdir(f"{tmp_dir}"): + local_file_path = os.path.join(tmp_dir,local_file) + local_file_modified_time = os.stat(os.path.join(tmp_dir,local_file)).st_mtime + + if (self._file_needs_update(local_file_path,local_file_modified_time)): + self.output_contract.publish_raw_file(local_file_path) + + def _file_needs_update(self,local_file_path,local_file_modified_time): + # Check if file needs to be pushed + # File is only considered to need to be pushed if it does not exist or has been modified since last push + output_contract = self.output_contract + + try: + s3_last_modified = output_contract.get_raw_file_metadata(local_file_path)['Metadata']['source_modified_time'] + if (float(s3_last_modified) < float(local_file_modified_time)): + return True + else: + return False + except: + return True diff --git a/core_project.yaml b/core_project.yaml index 2c881c62..951c792f 100644 --- a/core_project.yaml +++ b/core_project.yaml @@ -4,7 +4,7 @@ ENV_BUCKET: ichain-development ## constants DOCKER_REPO: ichain/core -DEV_BUCKET: ichain-development +DEV_BUCKET: ichain-dev-gluepoc PROD_BUCKET: ichain-production UAT_BUCKET: ichain-uat AWS_ACCOUNT: 687531504312 diff --git a/script/notebook b/script/notebook index 4b338dd7..fbfee731 100755 --- a/script/notebook +++ b/script/notebook @@ -15,4 +15,5 @@ docker run -it --rm \ -p 8888:8888 -p 4040:4040 -p 8080:8080 \ -e AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id) \ -e AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key) \ + -e AWS_DEFAULT_REGION=us-east-1 \ ichain/noteboook --allow-root --ip 0.0.0.0 --NotebookApp.token=''; diff --git a/setup.py b/setup.py index eba5503b..e83b07c1 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from distutils.extension import Extension PACKAGE_NAME = 'core' -MINIMUM_PYTHON_VERSION = '3.7' +MINIMUM_PYTHON_VERSION = '3.6' def check_python_version(): diff --git a/transforms/shared/raw/extract.ipynb b/transforms/shared/raw/extract.ipynb index 6fc1f498..b885877d 100644 --- a/transforms/shared/raw/extract.ipynb +++ b/transforms/shared/raw/extract.ipynb @@ -1,5 +1,15 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "!{sys.executable} -m pip install /host/project" + ] + }, { "cell_type": "code", "execution_count": null, @@ -11,9 +21,54 @@ "outputs": [], "source": [ "id = 1\n", - "input_contract = \"IM/AN/INPUT/CONTRACT/SHORT/AND/STOUT\"\n", - "output_contract = \"THIS/IS/MY/SPOUT\"\n", - "env = \"dev\"" + "\n", + "# input_contract = \"IM/AN/INPUT/CONTRACT/SHORT/AND/STOUT\"\n", + "# output_contract = \"THIS/IS/MY/SPOUT\"\n", + "env = \"dev\"\n", + "branch = \"DC-18-extract-main-executor\"\n", + "parent = \"test-manu2\"\n", + "child = \"test-brand\"\n", + "state = \"raw\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from core.helpers import configuration_mocker, file_mover\n", + "from core.models import configuration\n", + "from core import secret, contract\n", + "from core.transforms.shared.raw import extract\n", + "from pprint import pprint\n", + "from core.helpers.notebook import get_contract, get_transform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "transform = get_transform(id)\n", + "output_contract = get_contract(env=env,\n", + " state=state,\n", + " branch=branch,\n", + " parent=parent,\n", + " child=child\n", + " )\n", + "pprint(transform)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "extract_transform = extract.ExtractTransform(env=env, transform=transform, output_contract=output_contract)\n", + "extract_transform.run()" ] }, { From 7b66a4c33dc278dbe27260493cd16532c77ff16d Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 29 Jan 2019 13:06:11 -0500 Subject: [PATCH 3/9] added secret_type_of to extract_configuration --- core/helpers/configuration_mocker.py | 12 ++++++------ core/models/configuration.py | 1 + database/versions/c35c252fb0bc_.py | 26 ++++++++++++++++++++++++++ tests/unit/test_configuration.py | 1 + 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 database/versions/c35c252fb0bc_.py diff --git a/core/helpers/configuration_mocker.py b/core/helpers/configuration_mocker.py index 5acce72b..659c02e1 100644 --- a/core/helpers/configuration_mocker.py +++ b/core/helpers/configuration_mocker.py @@ -50,17 +50,17 @@ def _mock_extract_configurations(self)-> None: ex = config.ExtractConfiguration self.session.add_all([ ex(id=1, transformation_id=2, filesystem_path='', - prefix='', secret_name='dev-sftp'), + prefix='', secret_name='dev-sftp', secret_type_of='FTP'), ex(id=2, transformation_id=2, filesystem_path='banana_stand_data', - prefix='gob', secret_name='dev-sftp'), + prefix='gob', secret_name='dev-sftp', secret_type_of='FTP'), ex(id=3, transformation_id=3, filesystem_path='sudden_valley_holdings', - prefix='', secret_name='dev-sftp'), + prefix='', secret_name='dev-sftp', secret_type_of='FTP'), ex(id=4, transformation_id=1, filesystem_path='/incoming', - prefix='', secret_name='dev-sftp'), + prefix='', secret_name='dev-sftp', secret_type_of='FTP'), ex(id=5, transformation_id=1, filesystem_path='/incoming', - prefix='test-extract-root-prefix', secret_name='dev-sftp'), + prefix='test-extract-root-prefix', secret_name='dev-sftp', secret_type_of='FTP'), ex(id=6, transformation_id=1, filesystem_path='/incoming/testing_extract', - prefix='', secret_name='dev-sftp') + prefix='', secret_name='dev-sftp', secret_type_of='FTP') ]) self.session.commit() logging.debug('Done generating extract_configuration mocks.') diff --git a/core/models/configuration.py b/core/models/configuration.py index e0c0d3cf..74bd5ff3 100644 --- a/core/models/configuration.py +++ b/core/models/configuration.py @@ -80,6 +80,7 @@ class ExtractConfiguration(UniversalWithPrimary, Base): 'transformations.id'), nullable=False) filesystem_path = Column(String) prefix = Column(String) + secret_type_of = Column(String, nullable=False) secret_name = Column(String, nullable=False) transformation = relationship( "Transformation", back_populates='extract_configurations') diff --git a/database/versions/c35c252fb0bc_.py b/database/versions/c35c252fb0bc_.py new file mode 100644 index 00000000..07382814 --- /dev/null +++ b/database/versions/c35c252fb0bc_.py @@ -0,0 +1,26 @@ +"""empty message + +Revision ID: c35c252fb0bc +Revises: 7333d20cbb08 +Create Date: 2019-01-29 13:00:53.459520 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c35c252fb0bc' +down_revision = '7333d20cbb08' +branch_labels = None +depends_on = None + + +def upgrade(): + conn = op.get_bind() + conn.execute(" COMMENT ON COLUMN extract_configurations.secret_type_of IS 'represents the source type, eg. FTP, databse, S3 etc.';") + + +def downgrade(): + conn = op.get_bind() + conn.execute(" COMMENT ON COLUMN extract_configurations.secret_type_of IS '';") diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 5d982613..f0a06270 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -17,6 +17,7 @@ def test_get_extract_configuration(): for x in range(0, 3): sname = f'test_secret_{x}' session.add(ec(transformation_id=100, + secret_type_of="FTP", secret_name=sname)) test_secret_names.append(sname) From 19bb9ba2b78848dfba0701edb7b2c2e175829970 Mon Sep 17 00:00:00 2001 From: "Rayne N. Schaffer" Date: Tue, 29 Jan 2019 13:12:44 -0500 Subject: [PATCH 4/9] integrated new config field into extract class --- transforms/shared/raw/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/shared/raw/extract.py b/transforms/shared/raw/extract.py index ea1a4a97..53e162b2 100644 --- a/transforms/shared/raw/extract.py +++ b/transforms/shared/raw/extract.py @@ -47,7 +47,7 @@ def test_run(self): # Fetch secret from secret contract # TODO: Currently configs made for FTP only, FTP type passed in directly - source_secret = secret.Secret(name=self.config.secret_name,env=self.env,type_of="FTP",mode="write") + source_secret = secret.Secret(name=self.config.secret_name,env=self.env,type_of=self.config.secret_type_of,mode="write") # Get files from remote and start pushing to s3 with tempfile.TemporaryDirectory() as tmp_dir: From c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 29 Jan 2019 13:39:05 -0500 Subject: [PATCH 5/9] removed hardcoded test vals in contract --- tests/unit/test_contract.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_contract.py b/tests/unit/test_contract.py index 7075c716..6b2723b4 100644 --- a/tests/unit/test_contract.py +++ b/tests/unit/test_contract.py @@ -3,6 +3,7 @@ import boto3 import tempfile from core.helpers.project_root import ProjectRoot +from core.constants import DEV_BUCKET import moto import os @@ -10,7 +11,7 @@ def test_set_env_valid(): contract = Contract() contract.set_env('dev') - assert contract.env == 'ichain-development', 'failed to set dev environment' + assert contract.env == f'{DEV_BUCKET}', 'failed to set dev environment' def test_set_env_invalid(): @@ -42,14 +43,14 @@ def test_dataset_only(_contract): contract = _contract path = contract.get_s3_path() - assert path == 's3://ichain-development/master/merck/wonder_drug/ingest/valid_dataset/', 'path was incorrectly built.' + assert path == f's3://{DEV_BUCKET}/master/merck/wonder_drug/ingest/valid_dataset/', 'path was incorrectly built.' def test_with_partitions(_contract): contract = _contract contract.set_partitions(['partition_1', 'partition_2']) path = contract.get_s3_path() - assert path == 's3://ichain-development/master/merck/wonder_drug/ingest/valid_dataset/partition_1/partition_2/', 'path was incorrectly built with partitions.' + assert path == f's3://{DEV_BUCKET}/master/merck/wonder_drug/ingest/valid_dataset/partition_1/partition_2/', 'path was incorrectly built with partitions.' def test_with_file_name_with_partitions(_contract): @@ -57,7 +58,7 @@ def test_with_file_name_with_partitions(_contract): contract.set_file_name('29980385023509.parquet') contract.set_partitions(['partition_1', 'partition_2']) path = contract.get_s3_path() - assert path == 's3://ichain-development/master/merck/wonder_drug/ingest/valid_dataset/partition_1/partition_2/29980385023509.parquet', 'path was incorrectly built with partitions.' + assert path == f's3://{DEV_BUCKET}/master/merck/wonder_drug/ingest/valid_dataset/partition_1/partition_2/29980385023509.parquet', 'path was incorrectly built with partitions.' def test_file_name_no_partitions(_contract): @@ -65,7 +66,7 @@ def test_file_name_no_partitions(_contract): contract.partitions = [] # hack to jump around the 0 len guard contract.set_file_name('29980385023509.parquet') path = contract.get_s3_path() - assert path == 's3://ichain-development/master/merck/wonder_drug/ingest/valid_dataset/29980385023509.parquet', 'path was incorrectly built without partitions and with file name.' + assert path == f's3://{DEV_BUCKET}/master/merck/wonder_drug/ingest/valid_dataset/29980385023509.parquet', 'path was incorrectly built without partitions and with file name.' def test_quick_set(_contract): @@ -78,7 +79,7 @@ def test_quick_set(_contract): dataset='valid_dataset') assert contract.get_branch() == 'master', 'failed to set branch' - assert contract.get_env() == 'ichain-development', 'failed to set env' + assert contract.get_env() == f'{DEV_BUCKET}', 'failed to set env' assert contract.get_parent() == 'merck', 'failed to set parent' assert contract.get_child() == 'wonder_drug', 'failed to set parent' assert contract.get_state() == 'ingest', 'failed to set parent' @@ -217,7 +218,7 @@ def test_get_partition_size(): @moto.mock_s3 def _s3_mock_setup(): s3_client = boto3.client('s3') - s3_client.create_bucket(Bucket='ichain-development') + s3_client.create_bucket(Bucket=f'{DEV_BUCKET}') contract = Contract() contract = Contract(branch='master', @@ -242,7 +243,7 @@ def test_publish_raw_valid(): key = f'master/merck/wonder_drug/raw/{os.path.split(_file.name)[1]}' s3_client = boto3.client('s3') body = s3_client.get_object( - Bucket='ichain-development', Key=key)['Body'].read() + Bucket=f'{DEV_BUCKET}', Key=key)['Body'].read() assert body == text @@ -273,5 +274,5 @@ def test_publish_raw_metadata(): key = f'master/merck/wonder_drug/raw/{os.path.split(_file.name)[1]}' s3_client = boto3.client('s3') meta = s3_client.get_object( - Bucket='ichain-development', Key=key)['Metadata'] + Bucket=f'{DEV_BUCKET}', Key=key)['Metadata'] assert meta['source_modified_time'] == str(f_time) From 879decba9776b1f6242a732b456af532d4230643 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 29 Jan 2019 20:04:49 +0000 Subject: [PATCH 6/9] makin a test for extract! --- core/transforms/shared/raw/extract.py | 17 +- sqlalchemy. | 1252 +++++++++++++++++++++++++ tests/unit/test_extract.py | 63 ++ transforms/__init__.py | 0 transforms/shared/__init__.py | 0 transforms/shared/raw/__init__.py | 0 6 files changed, 1326 insertions(+), 6 deletions(-) create mode 100644 sqlalchemy. create mode 100644 tests/unit/test_extract.py create mode 100644 transforms/__init__.py create mode 100644 transforms/shared/__init__.py create mode 100644 transforms/shared/raw/__init__.py diff --git a/core/transforms/shared/raw/extract.py b/core/transforms/shared/raw/extract.py index 6c521f6b..11277de2 100644 --- a/core/transforms/shared/raw/extract.py +++ b/core/transforms/shared/raw/extract.py @@ -6,7 +6,13 @@ import tempfile class ExtractTransform(): - def __init__(self, env:str, transform, output_contract): + def __init__(self, env:str, transform: core.models.configuration.Transformation, output_contract: core.contract) -> None: + """ performs the extraction to a given output contract. + ARGS: + - env one of "dev", "prod", "UAT" + - transform a configuration contract instance + - output_contract a contract instance + """ self.env = env self.output_contract = output_contract self.transform = transform @@ -25,12 +31,11 @@ def run(self): # Get files from remote and start pushing to s3 with tempfile.TemporaryDirectory() as tmp_dir: file_mover.get_files(tmp_dir=tmp_dir,prefix=prefix,remote_path=remote_path,secret=source_secret) - self.push_to_s3(tmp_dir) + self.push_to_s3(tmp_dir, self.output_contract) - def push_to_s3(self, tmp_dir): - # Get the transformation's output contract - oc = self.output_contract + def push_to_s3(self, tmp_dir: str, output_contract: core.contract)-> None: + """ For a local file dir, push the file to s3 if it is newer or does not exist.""" # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints for local_file in os.listdir(f"{tmp_dir}"): @@ -38,7 +43,7 @@ def push_to_s3(self, tmp_dir): local_file_modified_time = os.stat(os.path.join(tmp_dir,local_file)).st_mtime if (self._file_needs_update(local_file_path,local_file_modified_time)): - self.output_contract.publish_raw_file(local_file_path) + output_contract.publish_raw_file(local_file_path) def _file_needs_update(self,local_file_path,local_file_modified_time): # Check if file needs to be pushed diff --git a/sqlalchemy. b/sqlalchemy. new file mode 100644 index 00000000..c635dd05 --- /dev/null +++ b/sqlalchemy. @@ -0,0 +1,1252 @@ +./database/alembic.ini:# directories, initial revisions must be specified with --version-path +./database/alembic.ini:# output_encoding = utf-8 +./database/alembic.ini:format = %(levelname)-5.5s [%(name)s] %(message)s +./database/versions/d8bfba24a470_baseline_migration.py:Create Date: 2019-01-20 17:45:22.930236 +./database/versions/d8bfba24a470_baseline_migration.py: # ### commands auto generated by Alembic - please adjust! ### +./database/versions/d8bfba24a470_baseline_migration.py: # ### commands auto generated by Alembic - please adjust! ### +./database/versions/c35c252fb0bc_.py:Create Date: 2019-01-29 13:00:53.459520 +./database/versions/7333d20cbb08_comments.py:Create Date: 2019-01-20 18:48:50.863496 +Binary file ./database/versions/__pycache__/.9df9a79a2998_init.cpython-37.pyc matches +Binary file ./database/versions/__pycache__/c35c252fb0bc_.cpython-37.pyc matches +Binary file ./database/versions/__pycache__/7333d20cbb08_comments.cpython-37.pyc matches +Binary file ./database/versions/__pycache__/1134cec79e78_audit_db_and_triggers.cpython-37.pyc matches +Binary file ./database/versions/__pycache__/9df9a79a2998_init.cpython-37.pyc matches +Binary file ./database/versions/__pycache__/d8bfba24a470_baseline_migration.cpython-37.pyc matches +./database/versions/1134cec79e78_audit_db_and_triggers.py:Create Date: 2019-01-20 17:57:27.820702 +./database/versions/1134cec79e78_audit_db_and_triggers.py: -- no index on the audit table. keep writes cheap. +./database/versions/1134cec79e78_audit_db_and_triggers.py: -- no constraints, minimize audit risk. +./database/versions/1134cec79e78_audit_db_and_triggers.py: -- PL for updated_at +./database/README:Generic single-database configuration. +./database/README.md:- the very first migration file needs to be run using the postgres DB for the connection. Following migrations will use the configuration\_application database. +./database/README.md:- the very first migration uses postgres role. subsiquent migrations use configurator role. +./core/models/configuration.py: RETURNS instance of sqlalchemy.orm.session.Session https://docs.sqlalchemy.org/en/rel_1_2/orm/tutorial.html#creating-a-session +./core/models/configuration.py: def __init__(self, engine: engine.base.Engine) -> None: +./core/models/configuration.py: def get_session(self) -> session.Session: +./core/models/configuration.py: def __init__(self, env: str, local: bool = False) -> None: +./core/models/configuration.py: def get_engine(self) -> engine.base.Engine: +./core/models/configuration.py: def _local_engine(self) -> engine.base.Engine: +./core/models/configuration.py: def _secret_defined_engine(self) -> engine.base.Engine: +./core/models/configuration.py: # TODO: in DC-57 update this to use secret +Binary file ./core/models/__pycache__/configuration.cpython-37.pyc matches +Binary file ./core/__pycache__/secret.cpython-37.pyc matches +Binary file ./core/__pycache__/configuration.cpython-37.pyc matches +Binary file ./core/__pycache__/contract.cpython-37.pyc matches +./core/transforms/shared/raw/extract.py: - env one of "dev", "prod", "UAT" +./core/transforms/shared/raw/extract.py: - transform +./core/secret.py: ''' Abstracts aws secretsmanager - values of the secret are callable attributes. ie: +./core/secret.py: def __init__(self, name: str = None, env: str = None, type_of: str = None, mode: str = None, identifier: str = None, force_env: bool = False) -> None: +./core/secret.py: - name (str): this is the human-readable name, also middle part of the fully formed secret contract +./core/secret.py: - env (str): one of dev, prod, uat, all +./core/secret.py: - type_of (str): one of FTP, database, s3 +./core/secret.py: - mode (str): one of read / write +./core/secret.py: - identifier (str): if defined this is the fully formed secret contract exactly as it is in secrets manager +./core/secret.py: - force_env (bool): if True, do not sub the universal secret (ie "all") when the given env is not found +./core/secret.py: def _parse_identifier(self, identifier: str) -> None: +./core/secret.py: def _build_identifier(self, env: str, type_of: str, name: str, mode: str) -> str: +./core/secret.py: def _get_secret(self, identifier: str, force_env: bool) -> str: +./core/secret.py: def _parse_secret(self, secret: dict) -> None: +./core/contract.py: *--------------------------------------------------------------------------------------------------------------------* +./core/contract.py: | s3:// {ENV} / {BRANCH} / {PARENT} / {CHILD} / {STATE} / {DATASET} / {PARTITION} [ / {SUB-PARTITION} / ] {FILENAME} | +./core/contract.py: *--------------------------------------------------------------------------------------------------------------------* +./core/contract.py: ENV - environment Must be one of development, uat, production. +./core/contract.py: Prefixed with integrichain- due to global unique reqirement +./core/contract.py: BRANCH - the software branch for development this will be the working pull request (eg pr-225) +./core/contract.py: PARENT - The top level source identifier +./core/contract.py: or another aggregator for future-proofing +./core/contract.py: CHILD - The sub level source identifier, generally the brand (and is aliased as such) +./core/contract.py: STATE - One of: raw, ingest, master, enhance, Enrich, Metrics +./core/contract.py: DATASET - The name of the collection of data. In an RDBMS this would be a table or view +./core/contract.py: PARTITION - for datasets, the partition is set in the prefix name +./core/contract.py: SUB-PARTITION - for datasets, the sub-partitions add additional partitioning with additional prefixes +./core/contract.py: FILENAME - nondescript in the contract +./core/contract.py: def get_branch(self)->str: +./core/contract.py: def get_parent(self)->str: +./core/contract.py: def get_state(self)->str: +./core/contract.py: def get_child(self)->str: +./core/contract.py: def get_env(self)->str: +./core/contract.py: def get_dataset(self)->str: +./core/contract.py: def get_partitions(self)->str: +./core/contract.py: def get_partition_size(self)->str: +./core/contract.py: def get_file_name(self)->str: +./core/contract.py: def get_contract_type(self)->str: +./core/contract.py: def get_previous_state(self)->str: +./core/contract.py: return self.STATES[cur - 1] +./core/contract.py: def get_next_state(self)->str: +./core/contract.py: if cur == len(self.STATES) - 1: +./core/contract.py: def set_branch(self, branch: str)->None: +./core/contract.py: def set_parent(self, parent: str)->None: +./core/contract.py: def set_child(self, child: str)->None: +./core/contract.py: def set_state(self, state: str)->None: +./core/contract.py: def set_dataset(self, dataset: str)->None: +./core/contract.py: def set_partition_size(self, size: int)->None: +./core/contract.py: def set_partitions(self, partitions: list)->None: +./core/contract.py: - partitions (list) an ordered list of partition names. +./core/contract.py: def set_file_name(self, file_name: str)->None: +./core/contract.py: def set_env(self, env: str)->None: +./core/contract.py: def _set_contract_type(self)->None: +./core/contract.py: ''' INTENT: sets what type of contract this is - file, partition, or dataset +./core/contract.py: def get_s3_path(self)->str: +./core/contract.py: def publish_raw_file(self, local_file_path: str) ->None: +./core/contract.py: '''accepts a local path to a file, publishes it as-is to s3 as long as state == raw.''' +./core/contract.py: def get_raw_file_metadata(self, local_file_path:str) ->None: +./core/contract.py: def get_brand(self)->str: +./core/contract.py: def get_customer(self)->str: +./core/contract.py: def set_brand(self, val)->None: +./core/contract.py: def set_customer(self)->None: +./core/contract.py: def get_bucket(self)->str: +./core/contract.py: def get_key(self)->str: +./core/contract.py: def set_bucket(self, env: str)->None: +./core/contract.py: def _validate_part(self, part: str)->str: +./core/helpers/file_mover.py: def is_dir(self, remote_file) -> bool: +./core/helpers/configuration_mocker.py: ''' for development, creates in-memory database instance and a matching session. +./core/helpers/configuration_mocker.py: def __init__(self)-> None: +./core/helpers/configuration_mocker.py: # this instansiates the in-memory sqlite instance +./core/helpers/configuration_mocker.py: def get_session(self) -> Session: +./core/helpers/configuration_mocker.py: def generate_mocks(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_brands(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_extract_configurations(self)-> None: +./core/helpers/configuration_mocker.py: prefix='', secret_name='dev-sftp', secret_type_of='FTP'), +./core/helpers/configuration_mocker.py: prefix='gob', secret_name='dev-sftp', secret_type_of='FTP'), +./core/helpers/configuration_mocker.py: prefix='', secret_name='dev-sftp', secret_type_of='FTP'), +./core/helpers/configuration_mocker.py: prefix='', secret_name='dev-sftp', secret_type_of='FTP'), +./core/helpers/configuration_mocker.py: prefix='test-extract-root-prefix', secret_name='dev-sftp', secret_type_of='FTP'), +./core/helpers/configuration_mocker.py: prefix='', secret_name='dev-sftp', secret_type_of='FTP') +./core/helpers/configuration_mocker.py: def _mock_pharmaceutical_companies(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_pipelines(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_pipeline_states(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_pipeline_state_types(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_pipeline_types(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_segments(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_transformations(self)-> None: +./core/helpers/configuration_mocker.py: def _mock_transformation_templates(self)-> None: +./core/helpers/project_root.py: def __init__(self) -> None: +./core/helpers/project_root.py: def find_setup_file(self, f: str) -> List[Union[str, bool]]: +./core/helpers/project_root.py: def __str__(self)-> List[Union[str, bool]]: +./core/helpers/project_root.py: def get_path(self)-> List[Union[str, bool]]: +Binary file ./core/helpers/__pycache__/configuration_mocker.cpython-37.pyc matches +Binary file ./core/helpers/__pycache__/docker.cpython-37.pyc matches +Binary file ./core/helpers/__pycache__/file_mover.cpython-37.pyc matches +Binary file ./core/helpers/__pycache__/s3_naming_helper.cpython-37.pyc matches +./core/helpers/docker.py:def build_image(full_tag: str) -> str: +./core/helpers/docker.py: digest_sha = repo_digest.split("@")[-1] +./core/helpers/docker.py: decoded_token = base64.b64decode(auth_data['authorizationToken']).decode("utf-8") +./core/helpers/docker.py:def get_aws_repository(full_tag: str, account_id: str) -> str: +./core/helpers/docker.py:# f"--id={transaction_id}", +./core/helpers/docker.py:# f"--input_contract={input_contract}", +./core/helpers/docker.py:# f"--output_contract={output_contract}", +./core/helpers/docker.py:# f"--env={env}", +./core/helpers/docker.py:# "--executer=batch" +./core/helpers/docker.py:# 'value': 'us-east-1' +./core/helpers/docker.py: job_role_arn: str = "arn:aws:iam::687531504312:role/ecs-tasks", +./core/helpers/docker.py:# De-register every revision of a job_def_name +./core/helpers/s3_naming_helper.py: def validate_part(self, value, allow_prefix: bool = True) -> tuple: +./core/helpers/s3_naming_helper.py: - value (str) the path or partial path in s3 +./core/helpers/s3_naming_helper.py: - allow_prefix (bool) if false will only validate single parts +./core/helpers/s3_naming_helper.py: def validate_bucket_name(self, bucket_name: str) -> tuple: +./core/helpers/s3_naming_helper.py: - bucket_name (str) the bucket name to validate +./core/helpers/s3_naming_helper.py: # must be between 3-63 chars +./core/helpers/s3_naming_helper.py: elif not bool(re.match(r"^[a-z0-9\-\.]*$", bucket_name)): +./core/helpers/s3_naming_helper.py: elif bucket_name.endswith('-'): +./core/helpers/s3_naming_helper.py: elif '.-' in bucket_name or '-.' in bucket_name: +./core/helpers/s3_naming_helper.py: elif bool(re.search(r"\.[^0-9a-z]*", bucket_name)): +./core/helpers/s3_naming_helper.py: def validate_s3_path(self, path: str)->tuple: +./core/helpers/s3_naming_helper.py: - path (str) the S3 path to validate +./core/helpers/s3_naming_helper.py: def _safe_chars(self) -> list: +./core/helpers/s3_naming_helper.py: safe += ['!', '-', '_', '.', '*', '(', ')', '='] +./core/helpers/notebook.py:def run_transform(env: str, id: int, input_contract: str, output_contract: str) -> str: +./core/helpers/notebook.py: # you can reach out to a db to pull the name this hard-coding will be replaced. +./core/helpers/notebook.py:def output_path(output_contract: str, transformation_name: str) -> str: +./core/helpers/notebook.py:def output_url(output_path: str) -> str: +./requirements.txt:python-coveralls==2.9.1 +./requirements.txt:#TODO as soon as master is merged move this to a pinned version 2018-01-23 +./.pytest_cache/README.md:which provides the `--lf` and `--ff` options, as well as the `cache` fixture. +Binary file ./dist/core-0.0.1-py3.7.egg matches +./core.egg-info/PKG-INFO:Metadata-Version: 1.1 +./core.egg-info/PKG-INFO:Home-page: UNKNOWN +./core.egg-info/PKG-INFO:Author-email: jdm@integrichain.com +./core.egg-info/PKG-INFO:Classifier: Development Status :: 1 - Planning +./core.egg-info/SOURCES.txt:core.egg-info/PKG-INFO +./core.egg-info/SOURCES.txt:core.egg-info/SOURCES.txt +./core.egg-info/SOURCES.txt:core.egg-info/dependency_links.txt +./core.egg-info/SOURCES.txt:core.egg-info/entry_points.txt +./core.egg-info/SOURCES.txt:core.egg-info/top_level.txt +./core_project.yaml:ENV_BUCKET: ichain-development +./core_project.yaml:DEV_BUCKET: ichain-dev-gluepoc +./core_project.yaml:PROD_BUCKET: ichain-production +./core_project.yaml:UAT_BUCKET: ichain-uat +./core_project.yaml:AWS_REGION: us-east-1 +./tests/unit/test_secret.py: mysql_secret_string = '{"user":"ic_db_user", "password":"uns3cur3", "host":"12345-host.aws.integrichain.net", "port":"3306", "database": "bluth", "rdbms":"mysql"}' +./tests/unit/test_secret.py: psql_read_secret_string = '{"user":"ic_db_user2", "password":"uns3cur3", "host":"22222-host.aws.integrichain.net", "port":"5432", "database": "configuration_application", "rdbms":"postgres", "schema":"public", "role":"public"}' +./tests/unit/test_secret.py: psql_write_secret_string = '{"user":"ic_db_user2", "password":"uns3cur3", "host":"22222-host.aws.integrichain.net", "port":"5432", "database": "configuration_application", "rdbms":"postgres", "schema":"public", "role":"configurator"}' +./tests/unit/test_moto_mocking.py: - moto is actually working for a given API +./tests/unit/test_moto_mocking.py: - calls are not actually hitting real AWS! +./tests/unit/test_moto_mocking.py: 'Body'].read().decode("utf-8") +Binary file ./tests/unit/__pycache__/test_docker_moto_debug.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_secret.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_contract.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_moto_mocking.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_configuration.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_helpers.cpython-37-PYTEST.pyc matches +Binary file ./tests/unit/__pycache__/test_notebook.cpython-37-PYTEST.pyc matches +./tests/unit/test_contract.py: bad_file_name = '@@$10-8953095830-.jpg' +./tests/unit/test_helpers.py: # depends on hard-coded values in mocker +./tests/unit/test_helpers.py: # depends on hard-coded values in mocker +./tests/unit/test_helpers.py: assert set(secrets) == set(['dev-sftp']) +./tests/unit/test_helpers.py: ## must be between 3-63 chars +./tests/unit/test_helpers.py: response = helper.validate_bucket_name('bucket-') +./tests/unit/test_helpers.py: response = helper.validate_bucket_name('bucket-.') +./tests/unit/test_helpers.py: assert response[0], f'failed to validate valid name - message {response[1]}' +Binary file ./tests/integration/__pycache__/test_docker.cpython-37-PYTEST.pyc matches +./tests/integration/test_docker.py: "corecli --help", +./tests/integration/test_docker.py: 'value': 'us-east-1' +./tests/integration/test_docker.py: digest_sha = repo_digest.split("@")[-1] +./tests/integration/test_docker.py: time_since_image_pushed = datetime.now(timezone.utc) - ecr_resp['imageDetails'][0]['imagePushedAt'] +./tests/integration/test_docker.py: # CannotPullContainerError: API error (404): manifest for 687531504312.dkr.ecr.us-east-1.amazonaws.com/ichain/core:it_test not found +Binary file ./tests/__pycache__/test_docker_moto_debug.cpython-37-PYTEST.pyc matches +Binary file ./tests/__pycache__/test_secret.cpython-37-PYTEST.pyc matches +Binary file ./tests/__pycache__/test_docker.cpython-37-PYTEST.pyc matches +Binary file ./tests/__pycache__/test_configuration.cpython-37-PYTEST.pyc matches +./docs/contract.md:The contract class defines the way we write to and read from the S3 data lake. It offers us an abstraction from directly writing to the filesystem, so we inject RDBMS-like structure to what is essentially a raw file store. +./docs/contract.md: *--------------------------------------------------------------------------------------------------------------------* +./docs/contract.md: | s3:// {ENV} / {BRANCH} / {PARENT} / {CHILD} / {STATE} / {DATASET} / {PARTITION} [ / {SUB-PARTITION} / ] {FILENAME} | +./docs/contract.md: *--------------------------------------------------------------------------------------------------------------------* +./docs/contract.md:This is enforced by using `getter` and `setter` arguments - i.e. `get\_state()` to retrieve a contract state, `set\_state('raw')` to set the state. +./docs/contract.md:get\_next\_state() and get\_previous\_state() refer to the standard transform order raw-\> dimensional +./docs/contract.md:get\_partition\_size() returns the number of MB for each partition (pre-compression) +./docs/dev_guide.md:This project uses the idea of having basic (mostly) scripts to do tasks like testing, building, and serving applications - this idea was taken from https://github.com/github/scripts-to-rule-them-all. These scripts live in the /script directory and MUST be executed from the base directory of this repository. +./docs/dev_guide.md:In development it can be useful to get the functionality without setting up and migrating a database, so we have a helper class that builds an in-memory sqlite instance and can populate mock data. +./docs/dev_guide.md:However, this is _not_ the same thing as the production database environment - Sqlite does not support our PL Postgres functions, triggers etc. To get this full functionality in development you have 2 options: +./docs/dev_guide.md:- point to a Development / UAT / Production configuration\_application instance: +./docs/dev_guide.md:- point to a local PG instance and run the migration suite: +./docs/dev_guide.md:- a running PG instance at localhost:5432 +./docs/dev_guide.md:- a database configuration\_application +./docs/dev_guide.md:- a pg user configurator with password configurator, who owns this database and the PUBLIC schema +./docs/dev_guide.md: >>>MAC: database YOU$ alembic revision -m "added default trigger" +./docs/dev_guide.md: ## auto-generate a completed DDL migration based on the diff between your model and the DB +./docs/dev_guide.md: >>>MAC: database YOU$ alembic revision --autogenerate -m "added table hamburger_salad" +./docs/dev_guide.md:login creds, host URLS, and other security-minded bits are managed by aws secretsmanager. To access these you can use the `Secret` class. +./docs/dev_guide.md:Note that env should be passed from some environment-aware variable. +./script/test:docker build -f $DIR/dockerfiles/python3-7.dockerfile . -t ichain/python:3.7 +./script/test:docker run --rm -it \ +./script/test: -e AWS_DEFAULT_REGION=us-east-1 \ +./script/test: -e AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id) \ +./script/test: -e AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key) \ +./script/test: -v $DIR:/usr/src/app \ +./script/test: -v /var/run/docker.sock:/var/run/docker.sock \ +./script/test: bash -c "coverage run --source core /usr/local/bin/pytest tests/unit && coverage report $1" +./script/citest:docker build -f $DIR/dockerfiles/python3-7.dockerfile . -t ichain/python:3.7 +./script/citest:docker run --rm -it \ +./script/citest: -e AWS_DEFAULT_REGION=us-east-1 \ +./script/citest: -e AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id) \ +./script/citest: -e AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key) \ +./script/citest: -v $DIR:/usr/src/app \ +./script/citest: -v /var/run/docker.sock:/var/run/docker.sock \ +./script/citest: bash -c "coverage run --source core /usr/local/bin/pytest tests/unit tests/integration && coverage report && coveralls" +./script/notebook:docker build -f $DIR/dockerfiles/notebook.dockerfile . -t ichain/noteboook +./script/notebook:docker run -it --rm \ +./script/notebook: -w /host \ +./script/notebook: -v /Users:/Users \ +./script/notebook: -v $HOME:/host/home \ +./script/notebook: -v $HOME/.aws:/root/.aws \ +./script/notebook: -v $DIR:/host/project \ +./script/notebook: -p 8888:8888 -p 4040:4040 -p 8080:8080 \ +./script/notebook: -e AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id) \ +./script/notebook: -e AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key) \ +./script/notebook: -e AWS_DEFAULT_REGION=us-east-1 \ +./script/notebook: ichain/noteboook --allow-root --ip 0.0.0.0 --NotebookApp.token=''; +./script/script_setup:if ! [[ $(which docker) && $(docker --version) ]]; then +./script/console:docker build -f $DIR/dockerfiles/python3-7.dockerfile . -t ichain/python:3.7 +./script/console:docker run --rm -it \ +./script/console: -v $DIR:/usr/src/app \ +./script/console: -v $HOME:/host/home \ +./script/console: -v /var/run/docker.sock:/var/run/docker.sock \ +./script/console: -e AWS_DEFAULT_REGION=us-east-1 \ +./script/console: -v $HOME:/root \ +./script/console: -e AWS_ACCESS_KEY_ID=$(aws --profile default configure get aws_access_key_id) \ +./script/console: -e AWS_SECRET_ACCESS_KEY=$(aws --profile default configure get aws_secret_access_key) \ +./setup.py: if parts[:-1] == [key, '=']: +./setup.py: return parts[-1].strip("'") +./setup.py: 'Development Status :: 1 - Planning', +./transforms/shared/raw/extract.ipynb: "!{sys.executable} -m pip install /host/project" +./transforms/shared/raw/extract.ipynb: "branch = \"DC-18-extract-main-executor\"\n", +./transforms/shared/raw/extract.ipynb: "parent = \"test-manu2\"\n", +./transforms/shared/raw/extract.ipynb: "child = \"test-brand\"\n", +./transforms/shared/raw/extract.ipynb: "mimetype": "text/x-python", +Binary file ./transforms/shared/raw/__pycache__/extract.cpython-37.pyc matches +./build/lib/core/models/configuration.py: RETURNS instance of sqlalchemy.orm.session.Session https://docs.sqlalchemy.org/en/rel_1_2/orm/tutorial.html#creating-a-session +./build/lib/core/models/configuration.py: def __init__(self, engine: engine.base.Engine) -> None: +./build/lib/core/models/configuration.py: def get_session(self) -> session.Session: +./build/lib/core/models/configuration.py: def __init__(self, env: str, local: bool = False) -> None: +./build/lib/core/models/configuration.py: def get_engine(self) -> engine.base.Engine: +./build/lib/core/models/configuration.py: def _local_engine(self) -> engine.base.Engine: +./build/lib/core/models/configuration.py: def _secret_defined_engine(self) -> engine.base.Engine: +./build/lib/core/models/configuration.py: # TODO: in DC-57 update this to use secret +./build/lib/core/secret.py: def __init__(self, name: str = None, env: str = None, type_of: str = None, mode: str = None, identifier: str = None, force_env: bool = False) -> None: +./build/lib/core/secret.py: - name (str): this is the human-readable name, also middle part of the fully formed secret contract +./build/lib/core/secret.py: - env (str): one of dev, prod, uat, all +./build/lib/core/secret.py: - type_of (str): one of FTP, database, s3 +./build/lib/core/secret.py: - mode (str): one of read / write +./build/lib/core/secret.py: - identifier (str): if defined this is the fully formed secret contract exactly as it is in secrets manager +./build/lib/core/secret.py: - force_env (bool): if True, do not sub the universal secret (ie "all") when the given env is not found +./build/lib/core/secret.py: def _parse_identifier(self, identifier: str) -> None: +./build/lib/core/secret.py: def _build_identifier(self, env: str, type_of: str, name: str, mode: str) -> str: +./build/lib/core/secret.py: def _get_secret(self, identifier: str, force_env: bool) -> str: +./build/lib/core/secret.py: def _parse_secret(self, secret: dict) -> None: +./build/lib/core/helpers/configuration_mocker.py: ''' for development, creates in-memory database instance and a matching session. +./build/lib/core/helpers/configuration_mocker.py: # this instansiates the in-memory sqlite instance +./build/lib/core/helpers/configuration_mocker.py: ex(id=6, transformation_id=1, filesystem_path='200-1', +./build/lib/core/helpers/project_root.py: def __init__(self) -> None: +./build/lib/core/helpers/project_root.py: def find_setup_file(self, f: str) -> List[Union[str, bool]]: +./build/lib/core/helpers/project_root.py: def __str__(self)-> List[Union[str, bool]]: +./build/lib/core/helpers/project_root.py: def get_path(self)-> List[Union[str, bool]]: +./build/lib/core/helpers/docker.py:def build_image(full_tag: str) -> str: +./build/lib/core/helpers/docker.py: digest_sha = repo_digest.split("@")[-1] +./build/lib/core/helpers/docker.py: decoded_token = base64.b64decode(auth_data['authorizationToken']).decode("utf-8") +./build/lib/core/helpers/docker.py:def get_aws_repository(full_tag: str, account_id: str) -> str: +./build/lib/tests/test_secret.py: mysql_secret_string = '{"user":"ic_db_user", "password":"uns3cur3", "host":"12345-host.aws.integrichain.net", "port":"3306", "database": "bluth", "rdbms":"mysql"}' +./build/lib/tests/test_secret.py: psql_read_secret_string = '{"user":"ic_db_user2", "password":"uns3cur3", "host":"22222-host.aws.integrichain.net", "port":"5432", "database": "configuration_application", "rdbms":"postgres", "schema":"public", "role":"public"}' +./build/lib/tests/test_secret.py: psql_write_secret_string = '{"user":"ic_db_user2", "password":"uns3cur3", "host":"22222-host.aws.integrichain.net", "port":"5432", "database": "configuration_application", "rdbms":"postgres", "schema":"public", "role":"configurator"}' +./build/lib/tests/test_helpers.py: # depends on hard-coded values in mocker +./build/lib/tests/test_helpers.py: # depends on hard-coded values in mocker +./build/lib/tests/test_docker.py: digest_sha = repo_digest.split("@")[-1] +./build/lib/tests/test_docker.py: time_since_image_pushed = datetime.now(timezone.utc) - ecr_resp['imageDetails'][0]['imagePushedAt'] +./.git/config: url = git@github.com:mgg-ic/core.git +Binary file ./.git/objects/61/098a42900f9a9e40df998705066213bd53a593 matches +Binary file ./.git/objects/0d/8331893c0dcf2b573cd7801d39ba7901c0d4d7 matches +Binary file ./.git/objects/95/2a2b672ef59c77deb0488e15e51e56403b0e6d matches +Binary file ./.git/objects/95/f8ba1d7f85d5eb92cd2638940777da4992f257 matches +Binary file ./.git/objects/95/e75a13674e5c6e963b1f8ac7097bfa40b05025 matches +Binary file ./.git/objects/59/30673ffe8dd96d280ad9d30599e920939b1860 matches +Binary file ./.git/objects/59/942c8cb3d9638c614226026ba4e6bb65577750 matches +Binary file ./.git/objects/0c/d10add1607e74481019ba489361fd9ae7434da matches +Binary file ./.git/objects/0c/fd28a96bd47f2112a0bd4cd187b5cfdecbbd40 matches +Binary file ./.git/objects/66/b18c5c9cc4fee1efde59a2d0180bf80fcede51 matches +Binary file ./.git/objects/66/2934bf93c0bb0ea13be2164b172d5353c73d30 matches +Binary file ./.git/objects/66/06a13c81636a847f141d13c56986d918ac7d20 matches +Binary file ./.git/objects/3e/7316b1d1f4eddc945f10916f5201dfd54cb505 matches +Binary file ./.git/objects/3e/2e06c00ac56f6204bdcf2bca9b5e8ad158bc2f matches +Binary file ./.git/objects/3e/2014b33b6e44fa6cc27fd17592d9ec4a0293a1 matches +Binary file ./.git/objects/3e/8177181de58656a381196f5d954dde99215492 matches +Binary file ./.git/objects/50/e49edaf3315df68f9e9b29ec0d7a6d56933d01 matches +Binary file ./.git/objects/50/bbc3486611bc4a30c68f1ee638d43eb151df8f matches +Binary file ./.git/objects/50/2bf105af272f1eb36dbbd9e3467d75932eb823 matches +Binary file ./.git/objects/68/f07bc98d1455aae1085baad6b18b16cc962a23 matches +Binary file ./.git/objects/68/2bf662181aee88ced91e4e2e6b782b1819e9fe matches +Binary file ./.git/objects/57/0e13d90121b973676e3b3e2847a5d2035e2d19 matches +Binary file ./.git/objects/57/c30473efc197ed6a00cb6352206721b0a267b9 matches +Binary file ./.git/objects/6f/c1f498ffe6c37e1b72d9cc24533fc8d9a91935 matches +Binary file ./.git/objects/6f/89fedfa853a635a479ceda89a2eb6d300b8c84 matches +Binary file ./.git/objects/6f/ead2c53b081debfd8af6a9ca3b3b66ae0208c4 matches +Binary file ./.git/objects/03/53cefbe010da9eca65a3f6133235b5f3e2e8d9 matches +Binary file ./.git/objects/03/04faada4a5f2c5e83bbd2252bb6085952c511b matches +Binary file ./.git/objects/9b/d49947177d8dcec9bd778371ca058d0f992829 matches +Binary file ./.git/objects/9b/b3563f4a45649564b70df2a543be03798f41ee matches +Binary file ./.git/objects/9b/aa851fbac79ac6c43048030977f5552ba9a4a4 matches +Binary file ./.git/objects/9e/b7bc42974d5aad3343c90180f79b9a269520e2 matches +Binary file ./.git/objects/9e/a88b1c0198c8a13d28e574a9b6d6131c850c4c matches +Binary file ./.git/objects/9e/5cee482a6184b02fdc29302d477dfd625d6707 matches +Binary file ./.git/objects/04/6cc20e74152054756145adfaa458de8acec63a matches +Binary file ./.git/objects/04/938960a2073ed46a8867e4061d8032b8830ec6 matches +Binary file ./.git/objects/32/f0bc1417fa1f1dcc385b67def69ec16ee9a209 matches +Binary file ./.git/objects/69/9154f124f7cfe76eb64b719ff216f9947d2733 matches +Binary file ./.git/objects/69/6778ad6485bf3f1e400eb43f84ba75d2fbe419 matches +Binary file ./.git/objects/3c/f5dfdbdd3e6d06c232ffb02844c063ff32ad57 matches +Binary file ./.git/objects/51/e2dcb63db1947238d29c8fbb626d3c3ce7354f matches +Binary file ./.git/objects/51/ed8b1259f6f09ac45f4333699d6bea656ef7d5 matches +Binary file ./.git/objects/3d/fd49c2f1f1f7c5bbae4977da1dbdfa9466672b matches +Binary file ./.git/objects/3d/0d0ecb9d0e9f77eb435075aa07261aada83e9e matches +Binary file ./.git/objects/3d/cad5c62e93fe7853936b395f00943596051910 matches +Binary file ./.git/objects/58/8f94a49d6eb48ac2859e733b12003eb75742b1 matches +Binary file ./.git/objects/67/223d19c01ffd0e1f6b8e2cdd2b451a57da0878 matches +Binary file ./.git/objects/0b/5f2691e2558cfbb8a5e63d7d92392c93051346 matches +Binary file ./.git/objects/93/89e2a9a29a2cd1e7003f5e36370f407817258c matches +Binary file ./.git/objects/93/b29ce499e8418fd964a2cc03b23ceb0f9a36ff matches +Binary file ./.git/objects/94/968a1ed3f734c51bde5286fb9495aab8311bec matches +Binary file ./.git/objects/0e/92db2c24ac795b1c658e104aa67d9c4f5bdb7e matches +Binary file ./.git/objects/60/95f417b55cedb2e82398b01ce65d4026fd6d77 matches +Binary file ./.git/objects/60/e534ca2868ee555f426cb5b712ea6f3a15b61c matches +Binary file ./.git/objects/60/de95538d6ab901130f106a94ecc2ab8f3e7d10 matches +Binary file ./.git/objects/60/9e2e074096d54aeab7a4d4a677e4b0f01dc603 matches +Binary file ./.git/objects/60/b6c6f2f2fbda81e74ea3e824f984f96e31c251 matches +Binary file ./.git/objects/60/a190fed6fb8c7e84435b1c071aef505bf3a738 matches +Binary file ./.git/objects/34/d94036e609e55a6080a685b3fa48fd0a9888d3 matches +Binary file ./.git/objects/5a/143a7b01b52cb47d4e4603d9f0b960b8c3bcfa matches +Binary file ./.git/objects/5a/5d1cd4e94ad479cb59760a19f4e8278e78f728 matches +Binary file ./.git/objects/5a/cce72b313499e84cc0a932c4fac2bc861e434c matches +Binary file ./.git/objects/5f/ae30209ce42f2bce282a6377a3ce38438d9565 matches +Binary file ./.git/objects/5f/6bad04e6cb624b559c36d58591df0ce6954200 matches +Binary file ./.git/objects/5f/ee4a4abf51284e308ee8139981f1e0cadfd8b5 matches +Binary file ./.git/objects/5f/ef1b0014b30f029b10dbb7f346c1440d02a284 matches +Binary file ./.git/objects/33/a83d0f86570e734454256baa15dd31f880f593 matches +Binary file ./.git/objects/05/bdb9890945daac6c81431d675eb0df81a7688a matches +Binary file ./.git/objects/05/fdb1544c0865e19878552ab2e19255172eacf5 matches +Binary file ./.git/objects/05/72c84d4e6ddcfb6a308cce272c4ff04585b499 matches +Binary file ./.git/objects/05/a3fa1e6a059e62aead2b9f03cf9926a5f81c01 matches +Binary file ./.git/objects/05/ceaa0d5f1f88acf27d5dced2dc05c7d6846029 matches +Binary file ./.git/objects/9d/daac467feda55d5bb36a7ccf71a94b02bc82ce matches +Binary file ./.git/objects/9d/94046c8203660cec1386932b5c682de518e204 matches +Binary file ./.git/objects/9d/350c3bec05d5159a62865fac256dffe5d6cf5f matches +Binary file ./.git/objects/02/747a11591ec6f9abfb4a20f9781baf19cab961 matches +Binary file ./.git/objects/a4/92bbec9745f73e4449289108b09e1101d7b77b matches +Binary file ./.git/objects/a4/5d3de5b41d79f984929054be149a2c24baa9e5 matches +Binary file ./.git/objects/a3/21751e5dde0b60981b6c2f27d3f4870c4c04d7 matches +Binary file ./.git/objects/a3/c5d6fde02adbaf2013ffe38b59f2db1a3f9bc6 matches +Binary file ./.git/objects/b5/0b2d6149a6a8065bd891aa7a4c0ceb48cd360e matches +Binary file ./.git/objects/b5/728c30f9c896b06a9c7e5860c58793e99a4439 matches +Binary file ./.git/objects/d9/94482783daf346d487e7a41bf3bca0968d2ed4 matches +Binary file ./.git/objects/d9/174f8831a12e90e2b0184d6dda9702a84b74fb matches +Binary file ./.git/objects/d9/234f0a36256f20670a5142e0e6fdff63cf6628 matches +Binary file ./.git/objects/d9/657a3e105ee6f9d4bcdd24dfc9fa88762d6069 matches +Binary file ./.git/objects/d9/66c125c97665b619f91448e5a237a3fe5f891a matches +Binary file ./.git/objects/ac/8e081b300920da042ce1b84af7e5708140e8a3 matches +Binary file ./.git/objects/ac/c8713c145024837b460186ed01d4b911fc5901 matches +Binary file ./.git/objects/ad/ce5844508930a03d1e82ac513ecadc76c702dc matches +Binary file ./.git/objects/bb/72d76f15e36ff2dd00d68267a14091e8394310 matches +Binary file ./.git/objects/bb/c0a2752c9f93f1a8eed59a4fb83dd7663d90e7 matches +Binary file ./.git/objects/d7/ffcc0a4ee09e9a8cf7719555ab9ad7ff88a3fa matches +Binary file ./.git/objects/d7/1e930741ea05ddc520963ab6144191cf35542d matches +Binary file ./.git/objects/d7/d0d89f697cc9a563114399f9c7e4b453da162e matches +Binary file ./.git/objects/d7/68f29a617cc11f368a87427f88eeb4cac994ee matches +Binary file ./.git/objects/d0/96b305ca9faaa0aafea747d671d97db0fc8aa8 matches +Binary file ./.git/objects/d0/37795a82e09bf26447c5ca7c962291d497244f matches +Binary file ./.git/objects/d0/e6e7cfabb9d78a67cdb93ba5d19e8679e0ea92 matches +Binary file ./.git/objects/be/1e5df4645494e49c94cfb5172181e95eb90f8b matches +Binary file ./.git/objects/be/4713f2d0235c5fd709850a4c73fb522df67d4f matches +Binary file ./.git/objects/df/8b7620b5fe3e33f0b21c24122fbeeb649fa1b7 matches +Binary file ./.git/objects/df/0a62d62e3c77c98f5b4d142b8a192531a9a75a matches +Binary file ./.git/objects/df/4b5fc17f22bf802db2f24c6ba88ed14687e564 matches +Binary file ./.git/objects/da/d33a45459058345131a42f8ed5a1e8817c9b88 matches +Binary file ./.git/objects/b4/887d95dfb8360ac038d9fbe7582d64eb2a1946 matches +Binary file ./.git/objects/b4/69010e807b47e336daa6ac23240ad7cee257dd matches +Binary file ./.git/objects/a5/cbf5c0a7a9e314a15799e39c5fa61635415dc1 matches +Binary file ./.git/objects/a5/7b51e53896f7324444f042f13c13345c17c2a2 matches +Binary file ./.git/objects/bd/6a87531a3be637948b38459ba1da826639dde1 matches +Binary file ./.git/objects/bd/78ed01cfc5ece2d76947da2741db903aac47d6 matches +Binary file ./.git/objects/bd/e1030b2b1a93fcc056df941c28755dd6bfd913 matches +Binary file ./.git/objects/bd/f494d53a0d863387b77e65a5b85925ca8f9142 matches +Binary file ./.git/objects/bd/b9bf62c9fcc987b5575168d9d98846a612682b matches +Binary file ./.git/objects/d1/898e18dcadbed6c4f8b6cf9c851637ce58c317 matches +Binary file ./.git/objects/d1/bb4893bccd87ae13e717dc136e23ac298022b3 matches +Binary file ./.git/objects/d1/2bd593130fbceb9ff0cadc031da35d5c89b43c matches +Binary file ./.git/objects/d1/6f7335ae32a0cbdef3b76902809528c2d6402d matches +Binary file ./.git/objects/d6/fec0f09cc5cbf193a550b2a96cb082886821d3 matches +Binary file ./.git/objects/bc/9950562c026d8f1e336fd3bc59ea1ceaf025f3 matches +Binary file ./.git/objects/bc/7860d927e724debe7515e2549e9e491e295fc6 matches +Binary file ./.git/objects/bc/b5a4c5720f9cbce4f0babe51cf63d78294bfae matches +Binary file ./.git/objects/ae/4da7f9d208feabecde8de4367eb398161bfda1 matches +Binary file ./.git/objects/d8/3f5306ec183f3ccbef16a296514a41d65d23ec matches +Binary file ./.git/objects/d8/0136cc5a0ac89e6c7ded439098fbd540cd9845 matches +Binary file ./.git/objects/ab/ef804a46a35edfba7260599bda5762dd4ec133 matches +Binary file ./.git/objects/e5/a08f3c829b7ffe430b78a923e90474c3d3c1c8 matches +Binary file ./.git/objects/e5/880af1569dab25a03cfb2b33e0b1371e63cf46 matches +Binary file ./.git/objects/e2/ef3095578374e30ed98f860d8bd356a16d27cf matches +Binary file ./.git/objects/f3/ef45704c6aded64943010415d4794ad06a6ce9 matches +Binary file ./.git/objects/eb/1bb9427d9917063f7532b7a864d554feb5292e matches +Binary file ./.git/objects/c7/b6846110764af55a2cc65b5dd5287ae6fb1a95 matches +Binary file ./.git/objects/c0/fefa36d5ddf3fbb24a683344e76f314527a598 matches +Binary file ./.git/objects/c0/1766c3fac821d953ad74185096d15ff5e3b974 matches +Binary file ./.git/objects/ee/63b04ed436b714cd9e339771b21c1a37c3aa2c matches +Binary file ./.git/objects/ee/6dad4b3e370af822cbe602e9477b8a0293035b matches +Binary file ./.git/objects/c9/47bb143b1894d967aa77c09138e1afd130b332 matches +Binary file ./.git/objects/fc/d3af0a3267ce6d119ac088f079db59b595cb51 matches +Binary file ./.git/objects/fc/a6e4d53a1c878b3ac590c605b7fe1ffdeb61ec matches +Binary file ./.git/objects/fc/bf2c52e2a66789eadb1f66ecf7bb8432a996b7 matches +Binary file ./.git/objects/fd/f21d52b6c7ee5529e0c2024e65262b2d56aec2 matches +Binary file ./.git/objects/fd/04af2bdcf31bdc736184dcaeaf06c15d83ce09 matches +Binary file ./.git/objects/fd/f1783114c1bb4d9ed5ae38c16ad46870e47dc4 matches +Binary file ./.git/objects/f2/b0418d2a33a1e8d0cd639f47ed60edf971f8bc matches +Binary file ./.git/objects/f2/b55dce6c09ead52c0df965f14705d9d32e5ea9 matches +Binary file ./.git/objects/f2/8a9696d20415d760c584a33d3ffdc7110480e3 matches +Binary file ./.git/objects/f2/deec00885baf1fc5332668faacc51581888bd1 matches +Binary file ./.git/objects/f5/79fe8cd80f1e8b3b3510ebbc97a2d2c6154d36 matches +Binary file ./.git/objects/e3/6b545903fcab4eb59ac8dabe68797dd43c3f63 matches +Binary file ./.git/objects/e3/50918645c37aaaee118ddd755b8e88bb06a532 matches +Binary file ./.git/objects/cf/ee7f08f26ce300f7df0eaa6e4876b2ac87936e matches +Binary file ./.git/objects/cf/ef9093ad063a5c1ffdc454b83de4d1816974fb matches +Binary file ./.git/objects/cf/849a43b8c67dcfb709925880329769d42f0959 matches +Binary file ./.git/objects/cf/efa1a131a213464942f22dc42bf3f2fe85cd2e matches +Binary file ./.git/objects/ca/6033fa7583756f1efa8b5ade72c9a01bd86f87 matches +Binary file ./.git/objects/ca/9a1b9958d6803ceb130289fa2d77942b5c27c8 matches +Binary file ./.git/objects/ca/d18a4933d754316d2b32e7819e25e20d266f28 matches +Binary file ./.git/objects/ca/fc8df1cd321e011c97371e71186cf3f68e73c3 matches +Binary file ./.git/objects/e4/c02357e96261e83347d7ccb2e13de7fcca792e matches +Binary file ./.git/objects/e4/2587afe43a0bdbb2a2fccc7528bfe0c96f69e1 matches +Binary file ./.git/objects/e4/8002044e96e069e799ff64bbc47080cf1b711a matches +Binary file ./.git/objects/e4/bef463339c0d39ac5dca6d4d4ec02622ab79b1 matches +Binary file ./.git/objects/fe/8addd48dd8f660a0afc3bb5f77fff590a28458 matches +Binary file ./.git/objects/fe/45a01f153730c994787aa23610372f9fd57e78 matches +Binary file ./.git/objects/fe/e1801892c4a8f73a68ce7d2b5065c063b2f401 matches +Binary file ./.git/objects/fe/20994fffc227bf7396d391d916e6edcb170a09 matches +Binary file ./.git/objects/c8/0682fecefd8a04673e6e3c8a2cbdd1e447a777 matches +Binary file ./.git/objects/c8/cf97a04168e7d5e89e0e24cfc81a25e6fab6d3 matches +Binary file ./.git/objects/c8/fc13819d22892e827cb83fd036f284ba71058d matches +Binary file ./.git/objects/c8/50a2cfe73c232955f770c996429e15f6896995 matches +Binary file ./.git/objects/fb/fee731c40b5e281c1724eaae0cb4552be19539 matches +Binary file ./.git/objects/fb/48d4431425a54e758bb17312969e84ce29643f matches +Binary file ./.git/objects/fb/93ebae652f5155e91d08618a8c608d9dfc896e matches +Binary file ./.git/objects/fb/2df4b2ae39fb8972ee53200f33744d17ecfe17 matches +Binary file ./.git/objects/fb/5d347a2556bdcedee4074d2d0098afc8c173a6 matches +Binary file ./.git/objects/ed/ef4a3781d26feeb6a80393836076ddc18a0563 matches +Binary file ./.git/objects/ed/288965648f4565d042b16be150d159b29bbb2e matches +Binary file ./.git/objects/ed/2d212b1960c21608d80ae71df7d8ef80c3ae27 matches +Binary file ./.git/objects/ed/77d5bffbf6061704bd44d025cd40bef205b5ca matches +Binary file ./.git/objects/c1/21e8f33d7556730e6f6a6b95214af1d18ee16a matches +Binary file ./.git/objects/c6/74c556479258332d27a5ff47ace25ec0c67e99 matches +Binary file ./.git/objects/ec/eb8725b8d1d8b7eedf14898daa1dfee05d021c matches +Binary file ./.git/objects/4e/2d6cef5bb7defdac2f041b9dde18627fdb3ffd matches +Binary file ./.git/objects/4e/7832dd79e896be0f5ef69f682dbc964e32888d matches +Binary file ./.git/objects/20/411b2078b2bdccc382d10d31b9c0fa41db5f3b matches +Binary file ./.git/objects/20/1323e3a37e8f9e6e0448117c9c7215ff3d8bb4 matches +Binary file ./.git/objects/20/34b47676284a340fc29f22bb4ab60487cb50dd matches +Binary file ./.git/objects/18/a0ad4dee6f39ceb8671bb52c45e5f15a3288aa matches +Binary file ./.git/objects/4b/57fb96fa7b47bcffec1244d8ae125c4f68a303 matches +Binary file ./.git/objects/4b/b3a4c11f2ad086aab92c915a63d72282c2a73f matches +Binary file ./.git/objects/pack/pack-9b40fa566f574be9f7c1747b1bce69480172708a.pack matches +Binary file ./.git/objects/pack/pack-bca41655e51489c058e3f406af6ca57b9216714a.idx matches +Binary file ./.git/objects/pack/pack-bca41655e51489c058e3f406af6ca57b9216714a.pack matches +Binary file ./.git/objects/pack/pack-9b40fa566f574be9f7c1747b1bce69480172708a.idx matches +Binary file ./.git/objects/11/8969e7ef44c4f1c188ecdc18f378a36f6a1bd2 matches +Binary file ./.git/objects/11/a01bfb5d92d1e6e32fc05588c568c03b1eb470 matches +Binary file ./.git/objects/11/92ab37c59060d78b272dfc710cd210c56bb653 matches +Binary file ./.git/objects/11/5661485519d8c390fd07fd9e2a34e6baefc060 matches +Binary file ./.git/objects/7d/0c211349eaf312ac599f9295d4fb691fcd6fd6 matches +Binary file ./.git/objects/7d/e65ba90b9bb41e04afec75547d2fa626bc2e57 matches +Binary file ./.git/objects/7c/dd22661d67bd80d20ec2e64d3eac2a8ccca12a matches +Binary file ./.git/objects/7c/8c000f5f501b431320ebfb3dac0e34bbf34923 matches +Binary file ./.git/objects/7c/d8ccd6fcf7d2f51ffc877e96fa578897ee0ce2 matches +Binary file ./.git/objects/7c/d46718925243aeefa3eb852a2a8ae7ef4f0fef matches +Binary file ./.git/objects/16/e81b7e07340bd807bd4569f5e5926cfe089711 matches +Binary file ./.git/objects/16/9423bcb79da65c40dbcd2e1e199fdbd1c4ce7f matches +Binary file ./.git/objects/16/6104c81dc4e04b198700c4e28eb6c8435ecc92 matches +Binary file ./.git/objects/42/fc44b7324bbdbb07f61c85aca3cd40c874b074 matches +Binary file ./.git/objects/42/2441adebd4926d0fa4303ad09ac04de1efef98 matches +Binary file ./.git/objects/42/ae70b1cdc8e051b181de538497b0ef2b041656 matches +Binary file ./.git/objects/89/1f53d1050e88ba886d23949bb7f6d700108075 matches +Binary file ./.git/objects/89/f7a2692562308e1c74a4ca1be3a0d2fc9b1b87 matches +Binary file ./.git/objects/45/ba895aaeff8a8b69b1505c777b12d9bd12f8ab matches +Binary file ./.git/objects/1f/a1409ed045dbac7db01310dad23e0f8f6d904f matches +Binary file ./.git/objects/1f/140ca7ba138c6b31013349da82b0b84a71dd17 matches +Binary file ./.git/objects/1f/4afee3764d935a0bc28c35c5dfbd3edaad9262 matches +Binary file ./.git/objects/87/edc99b45f60721e12c21945b087b2a1e8ffa61 matches +Binary file ./.git/objects/87/63b29d5651ec95da21d9cad9f8a7f28f37df35 matches +Binary file ./.git/objects/87/344fe729eb1f345c5c0416f9ebbc8265deb176 matches +Binary file ./.git/objects/80/8955de7131004bb897c81cf4a073f962c9823d matches +Binary file ./.git/objects/74/25ca6e3b5e97758e08b9038effda3f085c3c85 matches +Binary file ./.git/objects/74/ab9bc2ba05d7b3e276a64c894d2aecadecd225 matches +Binary file ./.git/objects/74/bd5ff3fa888fcc0e6817f21aca129eddc257a0 matches +Binary file ./.git/objects/28/917c8843e2df6cc32257e174e7ed7047425fd5 matches +Binary file ./.git/objects/7b/04ef5eaa2d762166f57e047918481960a59e2a matches +Binary file ./.git/objects/7b/66a4c33dc278dbe27260493cd16532c77ff16d matches +Binary file ./.git/objects/7b/801ebae62f388eeea9e919bf0b200302c19686 matches +Binary file ./.git/objects/8a/cc0d513cc2a9eb9eb4e5d8599fccaee21cd9a9 matches +Binary file ./.git/objects/7e/df890168bf0269a1e5fcd0c7da8fa7cab3f162 matches +Binary file ./.git/objects/7e/cac46ca40d7945c340ceac3a2700d4110e6168 matches +Binary file ./.git/objects/10/bc12b5c15389847c0b42ec55eaf46581f5f1f3 matches +Binary file ./.git/objects/19/9e77f79be024dfbb2680d6f66930cf2718f237 matches +Binary file ./.git/objects/19/47c147fd00757030c5dba1a2453563ab430140 matches +Binary file ./.git/objects/19/1a8148eb448f9e938d0b5c7c5e27542d2dcec7 matches +Binary file ./.git/objects/26/93d5443ca768861f1eca6d4084228a18a17e5c matches +Binary file ./.git/objects/26/2f78dea6cf894617eb626c8d7d67cbf7ae0f39 matches +Binary file ./.git/objects/26/f7f33cc714d4d49dd8c476e53f65a6e1c5623b matches +Binary file ./.git/objects/26/66d567b600fb679a6f272e11ea86d432ddfadf matches +Binary file ./.git/objects/4d/34496da718a6657b6e32e076c4e4029d6b6dca matches +Binary file ./.git/objects/4d/16e5f0dd230b9c796d283dd411b46288dd6bb9 matches +Binary file ./.git/objects/75/f47703fc1754213edeba7a4110d94ca9236422 matches +Binary file ./.git/objects/75/a151721358afe90ad91105877e94639276f18f matches +Binary file ./.git/objects/75/eab747a17528203c14386843d43fb4a0bbb052 matches +Binary file ./.git/objects/75/3808ce7857fe0a9c7be21b223d4adba91cddd7 matches +Binary file ./.git/objects/81/76f10958e0904bac85aa797f3d82075ddf616a matches +Binary file ./.git/objects/81/8697a9b7330ff0dfdedfa13f48e159d3181d31 matches +Binary file ./.git/objects/81/422950e73dde7ed0f404458fd4e2cdac67133d matches +Binary file ./.git/objects/81/ad77a94912ed54b22d84f81291c7271c0f76bc matches +Binary file ./.git/objects/81/7f4c3716a24b1d158f76c574335c2a902d2be1 matches +Binary file ./.git/objects/86/4ce945d9d0b60a1dda953d07c03012efa7979d matches +Binary file ./.git/objects/86/702c3864dcc6b722b388e84b81f67c1012086c matches +Binary file ./.git/objects/86/ee3b951af98b77bda53773ba4811c901877a6e matches +Binary file ./.git/objects/72/ac8f8e1b9addd89d729d4493acfdc7bad6136f matches +Binary file ./.git/objects/72/7499ae93f58bfc834a1a94d14e02adfc032522 matches +Binary file ./.git/objects/44/66bce1f1502106dc69a3edbe8d213cf7fefba5 matches +Binary file ./.git/objects/2a/bd081560c8ffb2ce85f12930e61e805c0583e5 matches +Binary file ./.git/objects/2a/b2503bb0412670c987eb9e9294bec4fa180a9f matches +Binary file ./.git/objects/43/1702cee0b9082ba5add28b1b8c5843fc7fbda4 matches +Binary file ./.git/objects/43/be3317a9807fa7199ce499f60598fa66c56930 matches +Binary file ./.git/objects/88/ba4d9190bb4714de23557384627e9278c61dd0 matches +Binary file ./.git/objects/9f/e23e2976466551e9b5c4d08f8741690542943f matches +Binary file ./.git/objects/9f/b403f3f8c5947d0d3625683ceb05b0182c12b7 matches +Binary file ./.git/objects/6b/2723b4facb476566934bca0267a33a11f182dc matches +Binary file ./.git/objects/6b/5ac7e1a0c5863283e202b79624aa326218b586 matches +Binary file ./.git/objects/6b/23179efe712d0c137d36929bd9dafba598f906 matches +Binary file ./.git/objects/07/382814a793c064815c6f17636dbb685f20752f matches +Binary file ./.git/objects/38/146e5d03b29367e0939a90601971d731fe7941 matches +Binary file ./.git/objects/38/4e5afde64a623a79ab6f3b699c5dfccdaa06c9 matches +Binary file ./.git/objects/00/04cc82bfed13de8ce2a37d1f0aea7f329fdbb6 matches +Binary file ./.git/objects/00/d6491f6637d2ad7b906ea7c5fb4643c4acf4b7 matches +Binary file ./.git/objects/6e/a171efb6a07a60846fcc983aeb33f4d5b71818 matches +Binary file ./.git/objects/9a/ad920c5e1c3786a74d93e47e35af3003c510cb matches +Binary file ./.git/objects/9a/f7a6e54e4f7a19ce6a415fa65e4813d84c87d2 matches +Binary file ./.git/objects/36/c012be1a1f3fdd7e9bbfa3a11c7e38e0f7d32b matches +Binary file ./.git/objects/36/b98f97d324b6ec2a5d5c2c2df9cf1347a6d6eb matches +Binary file ./.git/objects/36/41a314dbe6b0e1055d1270db06467853dc1490 matches +Binary file ./.git/objects/36/28554d14ff446c446a4b4899533dbd415b621a matches +Binary file ./.git/objects/5c/f3a471ad4faed0a4ada51ee1408cbb638d7f2b matches +Binary file ./.git/objects/5c/749cd52491a6fdb94ef6045118a85b2bd7fd34 matches +Binary file ./.git/objects/5c/2da410bde68b18dffd897f15b0f1423dc5addf matches +Binary file ./.git/objects/5c/f3ab81f1ca491e7ddd7dd8a2945b5b433edcdb matches +Binary file ./.git/objects/5c/8fb9e1a79e11ebc20470adbf9563d298899583 matches +Binary file ./.git/objects/09/b21e03c7f058af138ef3ae7767ee24477ef6d1 matches +Binary file ./.git/objects/09/3b5df637b6c22dc7b5ad3ccbdc95cf551ed8a5 matches +Binary file ./.git/objects/09/3ecbc23e8dcc2930eabb87aad4d61ef309923e matches +Binary file ./.git/objects/5d/fa7f5e99dc96e748dac4a66e7c19a8358ecad5 matches +Binary file ./.git/objects/5d/982613090c74dddca0375b5f6d779689a52e37 matches +Binary file ./.git/objects/31/9991374d19d9f02aa7f9ddaeb0da4ba1de000c matches +Binary file ./.git/objects/31/a404cb34e17970210f10bb1f2f81e1dcf9b4db matches +Binary file ./.git/objects/31/90f0a20770f59455bef20b0bbe4ec6535e3e5e matches +Binary file ./.git/objects/31/204c3145d26bb9b4418aa2a00c713415980c95 matches +Binary file ./.git/objects/91/ad5654b358b4b9c2ce89e1063a2be8236e0434 matches +Binary file ./.git/objects/65/703697dedabbe5b2279786b375534fca75889c matches +Binary file ./.git/objects/65/936aece7ca0e24721af23a2107d8a72934dfda matches +Binary file ./.git/objects/65/413c7cbb130e377c017e7babebcf443b62de5b matches +Binary file ./.git/objects/65/1d8d23123add4e03e850f80a9babe63a6b145a matches +Binary file ./.git/objects/65/9c02e1cd134afe50775f12212b9c246584f342 matches +Binary file ./.git/objects/62/89b984f61cb562d07e8d11a229fad4a412a105 matches +Binary file ./.git/objects/62/e7969121a6ce386ce93b2828dcc81fb51d9aee matches +Binary file ./.git/objects/96/7502e0a3a79d4ee8821a3539ea734320ab265f matches +Binary file ./.git/objects/3a/80e5b8952354d3497671ab3e0abd390bfd99e1 matches +Binary file ./.git/objects/3a/be398749a535d480ea5159bbb4f2d886a8b3ff matches +Binary file ./.git/objects/3a/b744e46f78d486c9e5e0ab88d714bb6abe63fd matches +Binary file ./.git/objects/98/e4f9c44effe479ed38c66ba922e7bcc672916f matches +Binary file ./.git/objects/98/f60d8fb82f8b8593d2516f69670cdf69fc95f1 matches +Binary file ./.git/objects/98/c76a04b75502bb59923804e33df29537b2f95e matches +Binary file ./.git/objects/98/590b9131c6f8f8e63e9a925ca766f6eab0fac1 matches +Binary file ./.git/objects/53/3bcae1b477034bd58a5394f7d7635c3a6b327f matches +Binary file ./.git/objects/53/e162b2eee03328708f375a0065c784dbe168d8 matches +Binary file ./.git/objects/53/e45104eba099de868017083ca4c0f77a9a6bc9 matches +Binary file ./.git/objects/53/6fead45e14a68f6f615697cb74aec46c769ffb matches +Binary file ./.git/objects/3f/343034d86bd05594cf8e188fa49a70d23e7c02 matches +Binary file ./.git/objects/3f/610abac2a461590d5ecfca88300ad03d9db464 matches +Binary file ./.git/objects/3f/399efe4bd9a21764a9237d33a20ecb239cde02 matches +Binary file ./.git/objects/30/b17e07cf39209a158dd70a4eb8a69c98f95175 matches +Binary file ./.git/objects/30/401daee3324868a7bfb4491a0605c6510afb68 matches +Binary file ./.git/objects/30/f240131571bddeb22356c7dd019babccc849d5 matches +Binary file ./.git/objects/5e/98a55aa39635bef63889e0e652bb49d29e1755 matches +Binary file ./.git/objects/5b/489e6d8dffe4c0bea04e1f9c4671376f7f18c4 matches +Binary file ./.git/objects/5b/8ed8d0217ef550f90830be4c7a56d7b3d507ba matches +Binary file ./.git/objects/37/6b5d39403e9a50d11d1f2beaa3b389deabdae8 matches +Binary file ./.git/objects/37/5936d222fd3a192a637d4bfbc839643e7fb58d matches +Binary file ./.git/objects/08/c4c21256328b9c307c9e892da29c981323d9c6 matches +Binary file ./.git/objects/6d/6bfffd2b119e102a431ecabd351622e44fd6d3 matches +Binary file ./.git/objects/6d/b0781910f1d4da512d12a5cad7ca3289689a5e matches +Binary file ./.git/objects/6d/db518d327a75e919e05116f24b106b6295a9e3 matches +Binary file ./.git/objects/6d/bcff4d4303440f963fbc4a33367760d7101026 matches +Binary file ./.git/objects/01/82ebd21372e5f84fa0aeb6d7c10f9f724002da matches +Binary file ./.git/objects/01/fbb74c1594f845f1626f73b9d3cf8274260830 matches +Binary file ./.git/objects/01/aa632e243b6f5f954e5a3f6d0a11a75f9e2f14 matches +Binary file ./.git/objects/01/1423dd05a8e544d54b07229743184724539963 matches +Binary file ./.git/objects/06/29153f1ce6aff7979a59dfe74ee6b70295cf4a matches +Binary file ./.git/objects/06/f2eaab61590f007c2fff50579eac5169df999a matches +Binary file ./.git/objects/06/6e471e8650caa3e4872c96157835ba9274605d matches +Binary file ./.git/objects/06/72959280840553c52494499c9c886b52575fb4 matches +Binary file ./.git/objects/6c/521f6b03c31739dbf6ee7e2a1cbac84d5f049c matches +Binary file ./.git/objects/6c/133299af75a2c2e465d8a66dfcc215c5cdaf84 matches +Binary file ./.git/objects/99/2ca3cf0547bac998eb98356bb30fc6f8c4a6c2 matches +Binary file ./.git/objects/99/96d3feca060dcb97d6d15834a06551d3d96a11 matches +Binary file ./.git/objects/99/c29f0720ec833215386c39ca58f8cfa6f7795e matches +Binary file ./.git/objects/99/3ca43931c0ddd48051dd52af0090899bdeaaaa matches +Binary file ./.git/objects/52/e5e8e499d481e0a93c7adf31523fc69e7a819f matches +Binary file ./.git/objects/52/4c3b463405ac15d6473c0c6dfe5e00e05458b4 matches +Binary file ./.git/objects/52/a5e7bc8368b724fbdc8465d96b87c40ee5742f matches +Binary file ./.git/objects/55/7c1ad49e7f62dbd774e9a79842c3a8249189da matches +Binary file ./.git/objects/55/b1f39fdf3902ec9bf304f7672aee3f68e4aa92 matches +Binary file ./.git/objects/55/947c6056980eaca563fecf73c3be6dbda24a21 matches +Binary file ./.git/objects/55/9752355e0f4040cbf467e29de28af5b41ee972 matches +Binary file ./.git/objects/55/f24bcfa75785055d5f17771780492d082a887e matches +Binary file ./.git/objects/63/d84b20907680adeb2a7b88cf9d7bd891e33a90 matches +Binary file ./.git/objects/0f/6ce276ee032ba27ec603d3cfe8e66c8e9d906d matches +Binary file ./.git/objects/0f/1a237bffc8ae4f3e3788e47ec12f03a3974e62 matches +Binary file ./.git/objects/0f/d8087a7068ffca53c7dc4c0a7752c56a1b0c0b matches +Binary file ./.git/objects/0a/688a0b824d050f6c7c5f84724b5c0e7f629abf matches +Binary file ./.git/objects/0a/86eb2daf340c0d78c7b21c3a2fe608ddb6a21e matches +Binary file ./.git/objects/64/778093367ee65085538b4ac9c00ca72fe0d8fc matches +Binary file ./.git/objects/64/9e6046123468f6fdb114a49032b70b0a8f5767 matches +Binary file ./.git/objects/64/8fe5d3c22ceadb769337e20fe699abec8e078d matches +Binary file ./.git/objects/64/e7e915edeee88ac07b99052a2909a624c953fe matches +Binary file ./.git/objects/64/b971078c59fdaf6d67f3f9c916d47113b9f377 matches +Binary file ./.git/objects/64/c70071291189a44d084b9d3794ddeafeb6bd18 matches +Binary file ./.git/objects/90/06b952b08fc27a6eaf1d62884139d8fc4841c5 matches +Binary file ./.git/objects/90/c7b95c0b4d8c948927be795993fa9a7669d9f1 matches +Binary file ./.git/objects/bf/8f0c628d2dd36cc1e640fc44b525e4714b65fb matches +Binary file ./.git/objects/bf/53d03bbf87d7d0a78977215405d429e4ffad84 matches +Binary file ./.git/objects/bf/cb1d9386ff264f04f2ce151a3a18aa62248686 matches +Binary file ./.git/objects/d3/93dabbf825c1c1ce9c2f922650d0864da1fc26 matches +Binary file ./.git/objects/d3/7a45bb7f4d8fb6e13c647afc6017ccb3ee840f matches +Binary file ./.git/objects/d4/cf1a66bf52b85cc620d610ab5d3c8dec03e264 matches +Binary file ./.git/objects/d4/641fc755b364d2fc9a8a91ca40e2585d48755d matches +Binary file ./.git/objects/ba/931bda3f1b4f3285ca24fe135bbe692921c271 matches +Binary file ./.git/objects/ba/6634bffc44d6e4affcb91c18d61f11aaf282c5 matches +Binary file ./.git/objects/ba/fd018f10d5fc807f35a61334075dbc8d6a190a matches +Binary file ./.git/objects/a0/146243f0bf9546ddf6165a445baa1ec9336123 matches +Binary file ./.git/objects/a0/8871075283ed2c840d434099c15532b4c1052c matches +Binary file ./.git/objects/a0/71f6d3f112eceb376edb51af51bcf921baab30 matches +Binary file ./.git/objects/b8/85877d0c764fe1944a22824706c37822634101 matches +Binary file ./.git/objects/b8/4eaed4a98f8f8ebc46bad463ec3e74e0810101 matches +Binary file ./.git/objects/b8/c4638df5d069eb9172c5aa47d62d3ef4711408 matches +Binary file ./.git/objects/dd/608740cf17b1ec48103cecf622cae9285444fa matches +Binary file ./.git/objects/dd/6473518ebadae428bc843f696d08471fa34301 matches +Binary file ./.git/objects/dd/b5d79bec63b1ee2c403125460f5332c23c08f9 matches +Binary file ./.git/objects/dd/64f51e3e891d715812d504b13bd158497d119e matches +Binary file ./.git/objects/dc/f38a59c94daa10d4bd721ad5f3cde855d723b2 matches +Binary file ./.git/objects/dc/999e155764893f9d6e79232589ceb33c75ebed matches +Binary file ./.git/objects/b6/71191952de9ab64905cdc0e026e2cf02d1584a matches +Binary file ./.git/objects/b6/5408bc7de6efca13529ec780462fc982e87bbb matches +Binary file ./.git/objects/a9/3d461a81274fa7015aa1c1ad3bb1af94010652 matches +Binary file ./.git/objects/a9/dff49f62de8fdfa6397b7f557e5dfdfaa24b7e matches +Binary file ./.git/objects/d5/64d0bc3dd917926892c55e3706cc116d5b165e matches +Binary file ./.git/objects/d5/7f059a003f3342c38eeb51a716a9b714f45e96 matches +Binary file ./.git/objects/d2/6a02b4f3512f2802e08576c00d2c4301f393a4 matches +Binary file ./.git/objects/d2/3e1d8b4b65455c15e0c126c29001f52b9f37f2 matches +Binary file ./.git/objects/d2/46519bd77403dbad4188cc4930b36fa933158f matches +Binary file ./.git/objects/d2/e7e9631c63fea8ef441205c60108ef4b0873b3 matches +Binary file ./.git/objects/aa/ecf069da9bae0acc4d2d789b91dabfc86eb4d2 matches +Binary file ./.git/objects/aa/ef608288038baafba709561209346ad14d005c matches +Binary file ./.git/objects/af/fc7f09f0aedf95ae53195aa16eb81e5b6421b5 matches +Binary file ./.git/objects/af/a0a83795a6632370e47a8267c1a199c9ba7732 matches +Binary file ./.git/objects/af/4dc9b1cbdd5627e94f37a50dd49683d0558d85 matches +Binary file ./.git/objects/af/dca0c3f33a7145a4cf15e12655d5c74d90fa2a matches +Binary file ./.git/objects/db/58ade0d2478c75b31832bf979bdc7725931e5e matches +Binary file ./.git/objects/db/b64dcca9f4d2f7bca99528180b099bf8fddfc9 matches +Binary file ./.git/objects/db/6cdf1eb03e28a588a2b408d42b73b2886a4e7d matches +Binary file ./.git/objects/a8/ef0e09fb5f7b87c3625e0fdfcbef9b37645804 matches +Binary file ./.git/objects/de/7525c6f0c0744bd347c762152c898f307266c2 matches +Binary file ./.git/objects/de/f8f937b45fef761a5496911e980472c992324f matches +Binary file ./.git/objects/a6/7028f1416340bb08b00e9dcada364f5ab12f4e matches +Binary file ./.git/objects/a6/91bbcf412bba7625292aed1286de560a67bc4c matches +Binary file ./.git/objects/a6/42a15011967aaa11aed17211506b78dfd4b029 matches +Binary file ./.git/objects/b9/9c8db59edf326583febd5872be34731358377c matches +Binary file ./.git/objects/a1/9019216e24c8fbbf8316e568fbc195b48e9a91 matches +Binary file ./.git/objects/ef/7ea1ee1086838229d07a89c4838d596edcebb0 matches +Binary file ./.git/objects/ef/daecb00a6e75f9a01fa9479cf27ce042f098a2 matches +Binary file ./.git/objects/ef/a6b6fc4f27ef939db19ed08f0401bddb463c7e matches +Binary file ./.git/objects/c3/7768cfd30c29b811925abc27744d14dad434e4 matches +Binary file ./.git/objects/c4/515f2135fdf8af44f8aa63baf66c484fe69e79 matches +Binary file ./.git/objects/c4/4848926a061b3fd690093cc8348629ffea537e matches +Binary file ./.git/objects/c4/c00534b58623f110438303dc6aab42d10b1f4b matches +Binary file ./.git/objects/ea/1a4a9775016556d7750b4cfa2c115edbdecbfb matches +Binary file ./.git/objects/ea/9a3169cd8152de92a1acf7b02386d97a4be895 matches +Binary file ./.git/objects/ea/456e649da208477cd8f42b9ca9f9476c15fcf9 matches +Binary file ./.git/objects/cd/e410edbc95b4e672f6839468963b5765408ef5 matches +Binary file ./.git/objects/cd/720a2e6881b364bc510d916f89fe7dff6c0f48 matches +Binary file ./.git/objects/cd/33d228016d63ac9f7f1112716fc2f67f56dedc matches +Binary file ./.git/objects/cd/6dc591579e026aab63406218e5e8884743c432 matches +Binary file ./.git/objects/cc/dd1f3304b2a1d36b8d13df40956c270b2b2ac4 matches +Binary file ./.git/objects/e6/3b540ac5ab18aa5e44a37fd311e6d170f7a34f matches +Binary file ./.git/objects/f0/94f73f732db94d7c23cd398f8fc18df2cc5ce2 matches +Binary file ./.git/objects/f0/effce071ceed5744c3c955c06abe8e8f50d09f matches +Binary file ./.git/objects/f0/5ae83741a8e4d80f0d5f640847d3fea84ca307 matches +Binary file ./.git/objects/f0/945a21695b6af7f3842efa9cbc3f94eb0ae1e6 matches +Binary file ./.git/objects/f0/a0627010f2ee708114fcc199ede0b88a81797f matches +Binary file ./.git/objects/f7/c3931861ef09a7b9a4b079faec99653414907f matches +Binary file ./.git/objects/f7/b2785c436114e067343443dc4932ae12bad835 matches +Binary file ./.git/objects/f7/16e72984af783abf933ef92f41a1a76cce4b85 matches +Binary file ./.git/objects/f7/d04a2dc30c07754ca575feee2b5bf1bfec2b7c matches +Binary file ./.git/objects/e8/3b07c12fc7bdd4a6cc27276eb6c2a23d51c362 matches +Binary file ./.git/objects/e8/5b7e0eaa0dc18e34498710dd9f2c28a14b128e matches +Binary file ./.git/objects/e8/f1ff13c9863c0732918a40191752e805929e40 matches +Binary file ./.git/objects/e8/0921cfb1d79328fce2076db020101a51c2447f matches +Binary file ./.git/objects/fa/c4de8c7a500a85c10f79d8bfae6a00959cbadd matches +Binary file ./.git/objects/ff/3cc87a4e285d91d7008870b400334696d3901d matches +Binary file ./.git/objects/c5/7370024bd144adc4b6059363e0a2b884e02426 matches +Binary file ./.git/objects/c5/8f395c3c015c9ae3b85071acd8f25e9bc3793f matches +Binary file ./.git/objects/c2/770af3e74312bc06ca83be4766dbe1ca72a61e matches +Binary file ./.git/objects/c2/a2b86a6beb84646c895a71539dd09a37e89d1b matches +Binary file ./.git/objects/c2/8e39b1f1edf24372edb2b124033e7314209d07 matches +Binary file ./.git/objects/f6/6bd116584a5ceca251f5aeaa919515f8681c0a matches +Binary file ./.git/objects/e9/0a7d2b42a86c73b68bc5f8915df4283fd551e1 matches +Binary file ./.git/objects/e9/76ac82bde2de01994afbebfc7de96ae7373481 matches +Binary file ./.git/objects/f1/88cde6ea7251d5e6d0e80faf6c1f9ab10ba479 matches +Binary file ./.git/objects/e7/5dd0ffd16062c479eb5f61165f98eb192967a1 matches +Binary file ./.git/objects/e7/a33f2189eb50192fe2764437a465d39f9b1494 matches +Binary file ./.git/objects/e7/9daf4b957d58ac13492ba96ccca9d7e9f3823c matches +Binary file ./.git/objects/cb/bb85b34ba309ae52ecabbe2069e1574828577d matches +Binary file ./.git/objects/ce/ae84a2338354cc2f55e29c441775ea4d8d4d3e matches +Binary file ./.git/objects/e0/c003b3a0c452f7786e281c40edffff39999f82 matches +Binary file ./.git/objects/e0/0cd624172bd9db8e85b2438221936ffd6f891b matches +Binary file ./.git/objects/e0/c0d3cf5a082b4f0db784631275bb552e39c817 matches +Binary file ./.git/objects/46/db4cb73623d76805fdf00b49d8b8f3430074e3 matches +Binary file ./.git/objects/2c/f2621ce44bb74b141ec24f2c83fdf222bcb367 matches +Binary file ./.git/objects/2c/881c62083a6e6388f28fd2a7513807de3c273d matches +Binary file ./.git/objects/2c/0124a2b6b62bf8fcb45a1b068a3ea341a6ac11 matches +Binary file ./.git/objects/2d/ba072ad4db3c8bb19a85d4cd6b09a2ee933e56 matches +Binary file ./.git/objects/41/d6b12e2f62695a3a8cdd00ebe361b1bb8230cb matches +Binary file ./.git/objects/41/f0c6e4c543440c34a7d3814c4012b73e726609 matches +Binary file ./.git/objects/41/af50efe2b3821713e845f67ace8dec35317840 matches +Binary file ./.git/objects/83/dd381fc1a29afe5752592ba84fa1a3c7ba81eb matches +Binary file ./.git/objects/83/6c1b91b354f8e54628e37ade93014b88e7c8e3 matches +Binary file ./.git/objects/83/ea542d4120bae51e767a6e75c071d65390ecec matches +Binary file ./.git/objects/83/5c5d536bddb31c35e3dc468d5374b3613451b4 matches +Binary file ./.git/objects/83/0adeb1f47c55e02fef5f9ba5d34c4f28b040ea matches +Binary file ./.git/objects/1b/af41a813657c3d0c1aed7e477eedd94cdcf653 matches +Binary file ./.git/objects/1b/a8c716bedf258b32f852647cf34edddbfee6bb matches +Binary file ./.git/objects/1b/b1bbf4beed74313540a53637d32edc6621311f matches +Binary file ./.git/objects/1b/f66cbd70958a02ec8421a4111dd4fc470a71e8 matches +Binary file ./.git/objects/1b/3ce6047edef6fb1deb0ba04366c9287b05304b matches +Binary file ./.git/objects/1b/5091af64678f8a9c0dda11d999d2e8cd98eed9 matches +Binary file ./.git/objects/1b/3aad4d8668f296e3de7f04dc5fad844c06391a matches +Binary file ./.git/objects/48/e981406fa9e7f1b06e5e3e2f1678b2ff83f744 matches +Binary file ./.git/objects/48/ba8eb8a72f32ff7d8182641fb76c5c8774f1e4 matches +Binary file ./.git/objects/48/56842079b25ffb7533a5a6e0c4e0964d671941 matches +Binary file ./.git/objects/48/a00dcb66bbbe9851aae41cfee57082f8ffb084 matches +Binary file ./.git/objects/70/d8a8fbb698fdb19e23d28dc061b1f9feb62dde matches +Binary file ./.git/objects/70/a5efdc17b7ff5e10464859aa1f1583c8f2a827 matches +Binary file ./.git/objects/70/75c716ae8e7fdab4e4366eba54bd60c499287a matches +Binary file ./.git/objects/1e/402e7b7db891f036b0ad4df0c0e409e7f416ac matches +Binary file ./.git/objects/1e/d76759d266c4f45eeb8b8d40777a800f254df5 matches +Binary file ./.git/objects/1e/445d18aad38e486194c2f14759b20d0b453465 matches +Binary file ./.git/objects/84/bb94f13d32676a149d66e234b00f8b6a1910ec matches +Binary file ./.git/objects/84/030c42b4854ac8c265ae54dd098b11f0bd1c10 matches +Binary file ./.git/objects/84/ae4d56cdffbf83a2baafa3af0d295c3427b095 matches +Binary file ./.git/objects/4a/825797e4ec55fd909100efc5078e8b64655176 matches +Binary file ./.git/objects/4a/54b3a0bff4d8500d9bbd51187ed2043e71691e matches +Binary file ./.git/objects/24/756276bdca657032989db442c8b6a5135a343e matches +Binary file ./.git/objects/23/0882d0efab49da1e166d1e484a6dbb6511e7c3 matches +Binary file ./.git/objects/4f/084488f3bebc2d14ef507b828db7e9456f3adf matches +Binary file ./.git/objects/4f/8c008502ca32c430d5c654b6cbf3a6be9b3b29 matches +Binary file ./.git/objects/8d/1204508d7a4bd14ae70a979879e96919e4b16b matches +Binary file ./.git/objects/8d/283d0e6479c5f90bf2017b49003b344f69a19f matches +Binary file ./.git/objects/15/b08904586ad0b9efbaa83d9dd5d69a9baa9772 matches +Binary file ./.git/objects/15/c0b40b25bd378e362caec4ee18d4160cdacfcc matches +Binary file ./.git/objects/15/9e92b43a7ce07f726e363f312cc4e6b30d6d1b matches +Binary file ./.git/objects/15/fe069d4f35a3589cda87715adc0fd4c6e150f0 matches +Binary file ./.git/objects/12/1dbd3b1a4bbdafbb668b1bef806144dc47f561 matches +Binary file ./.git/objects/8c/3d99d5043f7cb52b4c0ce1aa19e11f92c0aeb6 matches +Binary file ./.git/objects/8c/f3f309a9e3a0b1960ae428849b945718a040eb matches +Binary file ./.git/objects/8c/1e6618401c5c2082a64b65576c9eab607f1d07 matches +Binary file ./.git/objects/85/9ebbd715081c622c74bd721a59191187ff9800 matches +Binary file ./.git/objects/85/db5d71ec0ec60c528ba54bd7e9d7e70645ed97 matches +Binary file ./.git/objects/1d/811295f54db21f897913553b3ba2f2f6ec4865 matches +Binary file ./.git/objects/1d/9026a07dbfc7aadd67ba290ef7d176f1364614 matches +Binary file ./.git/objects/1d/93e9ce973a6dcf479c7fc2463ddb81e18353e1 matches +Binary file ./.git/objects/1d/465df5e99073bfd89acf7ee88f24a075b63910 matches +Binary file ./.git/objects/71/39edc217b3cf24f34fe65ba86c7d2f6f0c0de6 matches +Binary file ./.git/objects/71/01e6c0036441d83ebf76c0dc9af74c2f8e2fca matches +Binary file ./.git/objects/71/1ba6b4c6c07b5a08a911b24b87db6c8c10048c matches +Binary file ./.git/objects/71/d0afb76935edac29a86c6ac865ebfcc0d910de matches +Binary file ./.git/objects/71/156da1427c1418c8fabeea51345ddd3d049557 matches +Binary file ./.git/objects/71/c55e09dc9299f3871d19f7a62a996472e2c619 matches +Binary file ./.git/objects/76/3ab4ee8d8a712ec6841b07aa95807a631aab4d matches +Binary file ./.git/objects/76/bd3825f58c0a4a90ad9ba9bc037e1190d6d849 matches +Binary file ./.git/objects/82/0b1d692fa0e37f84854a7ec05b1349db66ac14 matches +Binary file ./.git/objects/49/9c69afdee15c85c83cca6887ff5df4af9c529a matches +Binary file ./.git/objects/2e/cdb68ce756db4856d3bb5b6d0b35caa0a0722c matches +Binary file ./.git/objects/2e/f2a670ff047648f7ac0e0e8695a7549151cad1 matches +Binary file ./.git/objects/2e/dc6fb72de73ab192de557b7b90c58bee826173 matches +Binary file ./.git/objects/2b/53a7f1cfa34ac5e992ff770b55cecc9f4a6422 matches +Binary file ./.git/objects/2b/3d9b58684c9840b88a73c7ffdac8d7c1fde1fb matches +Binary file ./.git/objects/2b/45379e2cb076cfddd4d212d397c45f0f4bb9fa matches +Binary file ./.git/objects/2b/5f28482767837b1b626d54c509c6a5a7c8b65a matches +Binary file ./.git/objects/47/e995f2dfdbe31ac4505d9c6e3b7608ab895112 matches +Binary file ./.git/objects/47/2e4c299d1d27fbefb2acc8d541734a2770817d matches +Binary file ./.git/objects/8b/2732a1eb4e713369c160f7561b5b364bfcac99 matches +Binary file ./.git/objects/8b/67744b78ef85a7fa7abcf26ed1f7dacdd8922c matches +Binary file ./.git/objects/8b/58c368e1e85bcdde6dab98bbb4d8dcf67b5cb6 matches +Binary file ./.git/objects/13/d307bfd1f2973f42eff48b68af90cc9971f9c5 matches +Binary file ./.git/objects/13/c070663f584891656c07146f506daffd1457df matches +Binary file ./.git/objects/7f/7ef9a84520717e8e68d8e504890af2f10b20f1 matches +Binary file ./.git/objects/7f/d2c57008afd3ec74888cb90d41644d423e4670 matches +Binary file ./.git/objects/7a/4d259586bebd5866a05f667df62be66bc302f1 matches +Binary file ./.git/objects/7a/7c6be8486d249657afc75e4cb2f913043481d1 matches +Binary file ./.git/objects/14/6c2bc44fd4f1a41920046058e296a1411c6552 matches +Binary file ./.git/objects/14/934eb670810b3c7bebea124dbce66c70e0ac0f matches +Binary file ./.git/objects/8e/2a5e97461b1f0f31b2193bdb3a98e46b9bf64a matches +Binary file ./.git/objects/22/744bdcbdaf5d4d05f3aaf2cdd9c700b74fc3aa matches +Binary file ./.git/objects/22/60bc6295cf5adfe0d6644b967672a516a81298 matches +./.git/HEAD:ref: refs/heads/DC-18-extract-main-executor +./.git/info/exclude:# git ls-files --others --exclude-from=.git/info/exclude +./.git/logs/HEAD:0000000000000000000000000000000000000000 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547606669 -0500 clone: from git@github.com:norton120/core.git +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547606893 -0500 checkout: moving from master to DC-71-configuration_database +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c 4c09fa4fd25c1e5dbdb4a36f194754b62468a1fb Ethan Knox 1547610483 -0500 commit: initial +./.git/logs/HEAD:4c09fa4fd25c1e5dbdb4a36f194754b62468a1fb 2dbc04dbbe9b23620599883dcc6233e989a48595 Ethan Knox 1547611115 -0500 commit: added readme +./.git/logs/HEAD:2dbc04dbbe9b23620599883dcc6233e989a48595 1960d18a2f91c4751ae844516a1f1e77daaa0b73 Ethan Knox 1547611170 -0500 commit: DC-71 +./.git/logs/HEAD:1960d18a2f91c4751ae844516a1f1e77daaa0b73 e8f1ff13c9863c0732918a40191752e805929e40 Ethan Knox 1547611678 -0500 commit: DC-71 #comment test comment +./.git/logs/HEAD:e8f1ff13c9863c0732918a40191752e805929e40 d0e31482327dddbcb3314734c813c058a365feec Ethan Knox 1547654622 -0500 commit: DC-71 #comment this is a test comment +./.git/logs/HEAD:d0e31482327dddbcb3314734c813c058a365feec 8efd5e3e4178b0c5de364eb6e28d8edee2e1036c Ethan Knox 1547655795 -0500 commit: #comment this worked from a fork +./.git/logs/HEAD:8efd5e3e4178b0c5de364eb6e28d8edee2e1036c f3ef45704c6aded64943010415d4794ad06a6ce9 Ethan Knox 1547655916 -0500 commit: DC-71 #comment this worked from a fork +./.git/logs/HEAD:f3ef45704c6aded64943010415d4794ad06a6ce9 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547656032 -0500 checkout: moving from DC-71-configuration_database to origin +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547656035 -0500 checkout: moving from c9bfd40daabdf7df72633fdb265987baed46ce5c to master +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547656063 -0500 checkout: moving from master to DC-57-secrets-library-module +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c 76d11346a77863af2935315b95bf9cf03fb0a5fd Ethan Knox 1547656220 -0500 commit: DC-57 #comment fork comment +./.git/logs/HEAD:76d11346a77863af2935315b95bf9cf03fb0a5fd d393dabbf825c1c1ce9c2f922650d0864da1fc26 Ethan Knox 1547656301 -0500 commit: #hold fork comment +./.git/logs/HEAD:d393dabbf825c1c1ce9c2f922650d0864da1fc26 3a80e5b8952354d3497671ab3e0abd390bfd99e1 Ethan Knox 1547656341 -0500 commit: DC-57 #hold fork comment +./.git/logs/HEAD:3a80e5b8952354d3497671ab3e0abd390bfd99e1 f3ef45704c6aded64943010415d4794ad06a6ce9 Ethan Knox 1547657155 -0500 checkout: moving from DC-57-secrets-library-module to DC-71-configuration_database +./.git/logs/HEAD:f3ef45704c6aded64943010415d4794ad06a6ce9 629396a80150254bf3ad82703078373c94dec36c Ethan Knox 1547665385 -0500 commit (merge): pulled added script structure from DC-30 +./.git/logs/HEAD:629396a80150254bf3ad82703078373c94dec36c 51b91ec9b4697c892457121b4d051741b90a3270 Ethan Knox 1547677699 -0500 commit: DC-71 #comment added tool to quick gen empty migration files. +./.git/logs/HEAD:51b91ec9b4697c892457121b4d051741b90a3270 cd6dc591579e026aab63406218e5e8884743c432 Ethan Knox 1547687277 -0500 commit: DC-71 #comment added local test support. +./.git/logs/HEAD:cd6dc591579e026aab63406218e5e8884743c432 3521796b4e8aebda8577a991c07ec7ef5434a2a6 Ethan Knox 1547691485 -0500 commit: break out migrations to logical actions +./.git/logs/HEAD:3521796b4e8aebda8577a991c07ec7ef5434a2a6 02af3ed5f77898895a64871ff20744f4dd6823f7 Ethan Knox 1547692642 -0500 commit: added segments +./.git/logs/HEAD:02af3ed5f77898895a64871ff20744f4dd6823f7 82aadfa2af8c31dd32c1e435eba68d1e3911f8d6 Ethan Knox 1547731481 -0500 commit: fixed constraint on brands +./.git/logs/HEAD:82aadfa2af8c31dd32c1e435eba68d1e3911f8d6 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547731486 -0500 checkout: moving from DC-71-configuration_database to master +./.git/logs/HEAD:c9bfd40daabdf7df72633fdb265987baed46ce5c a65b20c41d28324fe210f4afc198b48cfdeb570c Ethan Knox 1547731492 -0500 pull official master: Fast-forward +./.git/logs/HEAD:a65b20c41d28324fe210f4afc198b48cfdeb570c a65b20c41d28324fe210f4afc198b48cfdeb570c Ethan Knox 1547731672 -0500 checkout: moving from master to DC-29-config-importer-class +./.git/logs/HEAD:a65b20c41d28324fe210f4afc198b48cfdeb570c e4c02357e96261e83347d7ccb2e13de7fcca792e Ethan Knox 1547732684 -0500 commit: DC-29 #in-progress #comment need to sanity check SQLAlchemy against migrations +./.git/logs/HEAD:e4c02357e96261e83347d7ccb2e13de7fcca792e 5cecd4792dcc5894621dbf27da56fc2a81f5c32e Ethan Knox 1547757799 -0500 commit: sqlalchemy model builds. TODO: stack object and return compiled from class +./.git/logs/HEAD:5cecd4792dcc5894621dbf27da56fc2a81f5c32e b92b230b15f7c0e62666bd29493a95a07b4755fe Ethan Knox 1547763937 -0500 commit: DC-29 #comment sqlalchemy models structured to support extract_config, <= transform +./.git/logs/HEAD:b92b230b15f7c0e62666bd29493a95a07b4755fe 42ae70b1cdc8e051b181de538497b0ef2b041656 Ethan Knox 1547822372 -0500 commit: test coverage, moved mocks to helper +./.git/logs/HEAD:42ae70b1cdc8e051b181de538497b0ef2b041656 4fed023f7b3b524f032ea0330fc0352f328f7025 Ethan Knox 1547824165 -0500 commit: adding dev guide +./.git/logs/HEAD:4fed023f7b3b524f032ea0330fc0352f328f7025 2ecdb68ce756db4856d3bb5b6d0b35caa0a0722c Ethan Knox 1547825053 -0500 commit: DC-29 #comment pep8 cleanup, docs +./.git/logs/HEAD:2ecdb68ce756db4856d3bb5b6d0b35caa0a0722c 699154f124f7cfe76eb64b719ff216f9947d2733 Ethan Knox 1547826897 -0500 commit: docs again +./.git/logs/HEAD:699154f124f7cfe76eb64b719ff216f9947d2733 a45d3de5b41d79f984929054be149a2c24baa9e5 Ethan Knox 1547827277 -0500 commit: more pep8 +./.git/logs/HEAD:a45d3de5b41d79f984929054be149a2c24baa9e5 118969e7ef44c4f1c188ecdc18f378a36f6a1bd2 Ethan Knox 1547827347 -0500 commit: fixed doc whitespace issue +./.git/logs/HEAD:118969e7ef44c4f1c188ecdc18f378a36f6a1bd2 6ba129768cb955b1b0311e9006b0ebfe85b9e9c0 Ethan Knox 1547829952 -0500 commit: fixed TODO +./.git/logs/HEAD:6ba129768cb955b1b0311e9006b0ebfe85b9e9c0 a65b20c41d28324fe210f4afc198b48cfdeb570c Ethan Knox 1547833819 -0500 checkout: moving from DC-29-config-importer-class to master +./.git/logs/HEAD:a65b20c41d28324fe210f4afc198b48cfdeb570c eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1547833825 -0500 pull official master: Fast-forward +./.git/logs/HEAD:eb1bb9427d9917063f7532b7a864d554feb5292e 82aadfa2af8c31dd32c1e435eba68d1e3911f8d6 Ethan Knox 1547834325 -0500 checkout: moving from master to DC-71-configuration_database +./.git/logs/HEAD:82aadfa2af8c31dd32c1e435eba68d1e3911f8d6 20733536743b905526ddf7f8d48073509e7afb16 Ethan Knox 1547834420 -0500 commit (merge): merge conflicts +./.git/logs/HEAD:20733536743b905526ddf7f8d48073509e7afb16 4920dbd659a80e440969c98d3e1e4a6fee6ed925 Ethan Knox 1547842108 -0500 commit: migrations support ORM now +./.git/logs/HEAD:4920dbd659a80e440969c98d3e1e4a6fee6ed925 50bbc3486611bc4a30c68f1ee638d43eb151df8f Ethan Knox 1547849336 -0500 commit: added audit logging and updated_at triggers. Delete trigger still not setting delete correctly. TODO: fix that +./.git/logs/HEAD:50bbc3486611bc4a30c68f1ee638d43eb151df8f 7475ac4ad490951e0570098c39302a73c0da0308 Ethan Knox 1547871995 -0500 commit: DC-71 #comment migration handles pg-level updated_at, is_deleted and audit table +./.git/logs/HEAD:7475ac4ad490951e0570098c39302a73c0da0308 d9234f0a36256f20670a5142e0e6fdff63cf6628 Ethan Knox 1547942739 -0500 commit: drop soft delete support +./.git/logs/HEAD:d9234f0a36256f20670a5142e0e6fdff63cf6628 198329467fb35e24abe03ea11cf5686a95ba98e9 Ethan Knox 1547991151 -0500 commit: moved commons to mixins, fixed docs +./.git/logs/HEAD:198329467fb35e24abe03ea11cf5686a95ba98e9 c4c00534b58623f110438303dc6aab42d10b1f4b Ethan Knox 1548006974 -0500 commit: base alembic install +./.git/logs/HEAD:c4c00534b58623f110438303dc6aab42d10b1f4b a691bbcf412bba7625292aed1286de560a67bc4c Ethan Knox 1548017861 -0500 commit: updated model to reflect new ERD +./.git/logs/HEAD:a691bbcf412bba7625292aed1286de560a67bc4c 5b8ed8d0217ef550f90830be4c7a56d7b3d507ba Ethan Knox 1548023636 -0500 commit: helper supports full model definition +./.git/logs/HEAD:5b8ed8d0217ef550f90830be4c7a56d7b3d507ba 1d93e9ce973a6dcf479c7fc2463ddb81e18353e1 Ethan Knox 1548026587 -0500 commit: migrations and test coverage +./.git/logs/HEAD:1d93e9ce973a6dcf479c7fc2463ddb81e18353e1 cfefa1a131a213464942f22dc42bf3f2fe85cd2e Ethan Knox 1548028089 -0500 commit: docs +./.git/logs/HEAD:cfefa1a131a213464942f22dc42bf3f2fe85cd2e 31204c3145d26bb9b4418aa2a00c713415980c95 Ethan Knox 1548032472 -0500 commit: apply common comments on create +./.git/logs/HEAD:31204c3145d26bb9b4418aa2a00c713415980c95 6dbcff4d4303440f963fbc4a33367760d7101026 Ethan Knox 1548032722 -0500 commit: pep8 +./.git/logs/HEAD:6dbcff4d4303440f963fbc4a33367760d7101026 9469c3ee2361b01b46048f070bd825b0c6c4caa7 Ethan Knox 1548033365 -0500 commit: typing and pep8 +./.git/logs/HEAD:9469c3ee2361b01b46048f070bd825b0c6c4caa7 eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1548075533 -0500 checkout: moving from DC-71-configuration_database to master +./.git/logs/HEAD:eb1bb9427d9917063f7532b7a864d554feb5292e eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1548075603 -0500 checkout: moving from master to DC-57-secrets-module +./.git/logs/HEAD:eb1bb9427d9917063f7532b7a864d554feb5292e bcb348fa013ac3b955c7919c0f5155fd6d82199d Ethan Knox 1548077440 -0500 commit: DC-57 #in-progress #comment initial commit +./.git/logs/HEAD:bcb348fa013ac3b955c7919c0f5155fd6d82199d 9469c3ee2361b01b46048f070bd825b0c6c4caa7 Ethan Knox 1548084555 -0500 checkout: moving from DC-57-secrets-module to DC-71-configuration_database +./.git/logs/HEAD:9469c3ee2361b01b46048f070bd825b0c6c4caa7 e2ee96449983cf6a46aa5e2b1b3f0710e71c0617 Ethan Knox 1548084597 -0500 commit: cleanup +./.git/logs/HEAD:e2ee96449983cf6a46aa5e2b1b3f0710e71c0617 2a4f3a780bfe2da3fc170bd07a5e8d3679c024bc Ethan Knox 1548084813 -0500 commit: better name for GenerateEngine class +./.git/logs/HEAD:2a4f3a780bfe2da3fc170bd07a5e8d3679c024bc eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1548084851 -0500 checkout: moving from DC-71-configuration_database to master +./.git/logs/HEAD:eb1bb9427d9917063f7532b7a864d554feb5292e 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548084858 -0500 pull official master: Fast-forward +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548084874 -0500 checkout: moving from master to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 2a4f3a780bfe2da3fc170bd07a5e8d3679c024bc Ethan Knox 1548084891 -0500 checkout: moving from master to DC-71-configuration_database +./.git/logs/HEAD:2a4f3a780bfe2da3fc170bd07a5e8d3679c024bc f70369a0a518377968c4ecf2f3911a95670a6bf3 Ethan Knox 1548084944 -0500 commit (merge): merged upstream +./.git/logs/HEAD:f70369a0a518377968c4ecf2f3911a95670a6bf3 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548084961 -0500 checkout: moving from DC-71-configuration_database to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 3a80e5b8952354d3497671ab3e0abd390bfd99e1 Ethan Knox 1548084973 -0500 checkout: moving from master to DC-57-secrets-library-module +./.git/logs/HEAD:3a80e5b8952354d3497671ab3e0abd390bfd99e1 662934bf93c0bb0ea13be2164b172d5353c73d30 Ethan Knox 1548085071 -0500 pull origin master: Merge made by the 'recursive' strategy. +./.git/logs/HEAD:662934bf93c0bb0ea13be2164b172d5353c73d30 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548085279 -0500 checkout: moving from DC-57-secrets-library-module to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548085316 -0500 checkout: moving from master to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548085328 -0500 checkout: moving from master to DC-57-secret-module +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548085375 -0500 checkout: moving from DC-57-secret-module to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd f70369a0a518377968c4ecf2f3911a95670a6bf3 Ethan Knox 1548085382 -0500 checkout: moving from master to DC-71-configuration_database +./.git/logs/HEAD:f70369a0a518377968c4ecf2f3911a95670a6bf3 6b6d510027aa8ce8c84675c2ea8dda7474cbfc4a Ethan Knox 1548085521 -0500 commit: updated reqs +./.git/logs/HEAD:6b6d510027aa8ce8c84675c2ea8dda7474cbfc4a 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548085539 -0500 checkout: moving from DC-71-configuration_database to DC-57-secret-module +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 51ed8b1259f6f09ac45f4333699d6bea656ef7d5 Ethan Knox 1548089593 -0500 commit: DC-57 #comment tests +./.git/logs/HEAD:51ed8b1259f6f09ac45f4333699d6bea656ef7d5 1eeeec5330a3c895e73bff6faf3b3be0b3014c06 Ethan Knox 1548091685 -0500 commit: method signature +./.git/logs/HEAD:1eeeec5330a3c895e73bff6faf3b3be0b3014c06 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548091689 -0500 checkout: moving from DC-57-secret-module to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 6b6d510027aa8ce8c84675c2ea8dda7474cbfc4a Ethan Knox 1548091697 -0500 checkout: moving from master to DC-71-configuration_database +./.git/logs/HEAD:6b6d510027aa8ce8c84675c2ea8dda7474cbfc4a a9dff49f62de8fdfa6397b7f557e5dfdfaa24b7e Ethan Knox 1548092082 -0500 commit: fixed merge straggler +./.git/logs/HEAD:a9dff49f62de8fdfa6397b7f557e5dfdfaa24b7e 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548092939 -0500 checkout: moving from DC-71-configuration_database to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 1eeeec5330a3c895e73bff6faf3b3be0b3014c06 Ethan Knox 1548092942 -0500 checkout: moving from master to DC-57-secret-module +./.git/logs/HEAD:1eeeec5330a3c895e73bff6faf3b3be0b3014c06 3082cf3c219d99556b44a3387877728e360b274b Ethan Knox 1548108635 -0500 commit: added universal secret support +./.git/logs/HEAD:3082cf3c219d99556b44a3387877728e360b274b 1bdefbeb3c95453b72d2f69008636738394d5b1e Ethan Knox 1548194637 -0500 commit: Tests pass on the metal. Tests fail in Docker, for some reason not mocking and actually calling aws. +./.git/logs/HEAD:1bdefbeb3c95453b72d2f69008636738394d5b1e da650db4c28a5a9d03882ba36cb55bffc0c00fce Ethan Knox 1548268959 -0500 commit: debugging moto +./.git/logs/HEAD:da650db4c28a5a9d03882ba36cb55bffc0c00fce c8fc13819d22892e827cb83fd036f284ba71058d Ethan Knox 1548269010 -0500 commit (merge): merged master +./.git/logs/HEAD:c8fc13819d22892e827cb83fd036f284ba71058d 4cc64ae1120b47bf9393e779aac041904bce2219 Ethan Knox 1548274697 -0500 pull official master: Merge made by the 'recursive' strategy. +./.git/logs/HEAD:4cc64ae1120b47bf9393e779aac041904bce2219 7fb395a0d6de0cd0971691f431b9f59ef922ac74 Ethan Knox 1548275820 -0500 commit: updated requirements.txt moto to support mocking. test coverage +./.git/logs/HEAD:7fb395a0d6de0cd0971691f431b9f59ef922ac74 53e45104eba099de868017083ca4c0f77a9a6bc9 Ethan Knox 1548275947 -0500 commit: mend +./.git/logs/HEAD:53e45104eba099de868017083ca4c0f77a9a6bc9 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548275960 -0500 checkout: moving from DC-57-secret-module to master +./.git/logs/HEAD:2034b47676284a340fc29f22bb4ab60487cb50dd 70a5efdc17b7ff5e10464859aa1f1583c8f2a827 Ethan Knox 1548275966 -0500 pull official master: Fast-forward +./.git/logs/HEAD:70a5efdc17b7ff5e10464859aa1f1583c8f2a827 53e45104eba099de868017083ca4c0f77a9a6bc9 Ethan Knox 1548275979 -0500 checkout: moving from master to DC-57-secret-module +./.git/logs/HEAD:53e45104eba099de868017083ca4c0f77a9a6bc9 3b95be97f03659c597dd7dbefb5e481f314c5569 Ethan Knox 1548276364 -0500 commit: comments +./.git/logs/HEAD:3b95be97f03659c597dd7dbefb5e481f314c5569 86ee3b951af98b77bda53773ba4811c901877a6e Ethan Knox 1548276686 -0500 commit: docs +./.git/logs/HEAD:86ee3b951af98b77bda53773ba4811c901877a6e 3f52c8915d3da0baa95a49905080b24360206018 Ethan Knox 1548277488 -0500 commit: pinned moto to ic fork +./.git/logs/HEAD:3f52c8915d3da0baa95a49905080b24360206018 70a5efdc17b7ff5e10464859aa1f1583c8f2a827 Ethan Knox 1548336528 -0500 checkout: moving from DC-57-secret-module to master +./.git/logs/HEAD:70a5efdc17b7ff5e10464859aa1f1583c8f2a827 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548336532 -0500 pull official master: Fast-forward +./.git/logs/HEAD:f2deec00885baf1fc5332668faacc51581888bd1 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548336740 -0500 checkout: moving from master to DC-121-port-contract +./.git/logs/HEAD:f2deec00885baf1fc5332668faacc51581888bd1 98132e1f80857ece162ac84b2891a4e3abe8bea9 Ethan Knox 1548337802 -0500 commit: #DC-121 #Working s3 name helper +./.git/logs/HEAD:98132e1f80857ece162ac84b2891a4e3abe8bea9 0629153f1ce6aff7979a59dfe74ee6b70295cf4a Ethan Knox 1548346971 -0500 commit: s3 helper coverage +./.git/logs/HEAD:5cf3a471ad4faed0a4ada51ee1408cbb638d7f2b 168dcbd44157cb6375d70c6c84053e29cf4cfaf6 Ethan Knox 1548357371 -0500 commit: added coverage line flag to script +./.git/logs/HEAD:168dcbd44157cb6375d70c6c84053e29cf4cfaf6 85946408d0e378f495fda670da15839130d6bff2 Ethan Knox 1548357674 -0500 commit: coverage +./.git/logs/HEAD:85946408d0e378f495fda670da15839130d6bff2 8cf3f309a9e3a0b1960ae428849b945718a040eb Ethan Knox 1548364437 -0500 commit: cleaned up unused methods +./.git/logs/HEAD:8cf3f309a9e3a0b1960ae428849b945718a040eb 0000000000000000000000000000000000000000 Ethan Knox 1548364533 -0500 Branch: renamed refs/heads/DC-121-port-contract to refs/heads/Dc-128-port-contract +./.git/logs/HEAD:0000000000000000000000000000000000000000 8cf3f309a9e3a0b1960ae428849b945718a040eb Ethan Knox 1548364533 -0500 Branch: renamed refs/heads/DC-121-port-contract to refs/heads/Dc-128-port-contract +./.git/logs/HEAD:8cf3f309a9e3a0b1960ae428849b945718a040eb d1ea4b40f44cd4784e5b70826a2d6f7c912db3d1 Ethan Knox 1548364712 -0500 commit: autopep8 +./.git/logs/HEAD:d1ea4b40f44cd4784e5b70826a2d6f7c912db3d1 1cb6c90a1f9fb76e9fa5a46277cdc6ebd82c7a56 Ethan Knox 1548365295 -0500 commit: cleanup coverage junk +./.git/logs/HEAD:1cb6c90a1f9fb76e9fa5a46277cdc6ebd82c7a56 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548365507 -0500 checkout: moving from Dc-128-port-contract to master +./.git/logs/HEAD:f2deec00885baf1fc5332668faacc51581888bd1 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548366170 -0500 checkout: moving from master to master +./.git/logs/HEAD:f2deec00885baf1fc5332668faacc51581888bd1 1cb6c90a1f9fb76e9fa5a46277cdc6ebd82c7a56 Ethan Knox 1548366175 -0500 checkout: moving from master to Dc-128-port-contract +./.git/logs/HEAD:1cb6c90a1f9fb76e9fa5a46277cdc6ebd82c7a56 87e1cda3e3224058a5f487043d64a4cab3a24d8a Ethan Knox 1548367515 -0500 commit: downcase + stub for raw publish method +./.git/logs/HEAD:87e1cda3e3224058a5f487043d64a4cab3a24d8a f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548367564 -0500 checkout: moving from Dc-128-port-contract to master +./.git/logs/HEAD:f2deec00885baf1fc5332668faacc51581888bd1 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548367571 -0500 pull official master: Fast-forward +./.git/logs/HEAD:557c1ad49e7f62dbd774e9a79842c3a8249189da 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548426606 -0500 checkout: moving from master to DC-39-contract-getter-setter +./.git/logs/HEAD:557c1ad49e7f62dbd774e9a79842c3a8249189da a8dee17dd2708596a8c67ac7825ed40ee54279d8 Ethan Knox 1548447398 -0500 commit: invalid test +./.git/logs/HEAD:a8dee17dd2708596a8c67ac7825ed40ee54279d8 d4641fc755b364d2fc9a8a91ca40e2585d48755d Ethan Knox 1548447437 -0500 commit: autopep8 +./.git/logs/HEAD:d4641fc755b364d2fc9a8a91ca40e2585d48755d 2260bc6295cf5adfe0d6644b967672a516a81298 Ethan Knox 1548448339 -0500 commit: docs +./.git/logs/HEAD:2260bc6295cf5adfe0d6644b967672a516a81298 44c17d4606362784c4e45f99f579f429a5117892 Ethan Knox 1548448825 -0500 commit: meta test +./.git/logs/HEAD:44c17d4606362784c4e45f99f579f429a5117892 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548449164 -0500 checkout: moving from DC-39-contract-getter-setter to master +./.git/logs/HEAD:557c1ad49e7f62dbd774e9a79842c3a8249189da ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548449177 -0500 pull official master: Fast-forward +./.git/logs/HEAD:ea9a3169cd8152de92a1acf7b02386d97a4be895 ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548449219 -0500 checkout: moving from master to DC-131-constants +./.git/logs/HEAD:ea9a3169cd8152de92a1acf7b02386d97a4be895 e5cd40e9e96720e9cf31d44d8e4ea019faad5103 Ethan Knox 1548451427 -0500 commit: updated hardcoded vals to read config +./.git/logs/HEAD:e5cd40e9e96720e9cf31d44d8e4ea019faad5103 36c012be1a1f3fdd7e9bbfa3a11c7e38e0f7d32b Ethan Knox 1548454373 -0500 commit: moved to constants class +./.git/logs/HEAD:36c012be1a1f3fdd7e9bbfa3a11c7e38e0f7d32b f5f91da71588e96d66892a024c65afbc9bd684db Ethan Knox 1548454526 -0500 pull official master: Merge made by the 'recursive' strategy. +./.git/logs/HEAD:f5f91da71588e96d66892a024c65afbc9bd684db ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548692449 -0500 checkout: moving from DC-131-constants to master +./.git/logs/HEAD:ea9a3169cd8152de92a1acf7b02386d97a4be895 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548692476 -0500 pull official master: Fast-forward +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548696812 -0500 checkout: moving from master to master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548696825 -0500 checkout: moving from master to DC-60-dbt-initial-setup +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 6d6bfffd2b119e102a431ecabd351622e44fd6d3 Ethan Knox 1548702392 -0500 commit: docs +./.git/logs/HEAD:6d6bfffd2b119e102a431ecabd351622e44fd6d3 e3e8c3f2a0fe4575d565013292023584688036e3 Ethan Knox 1548704534 -0500 commit: svg support +./.git/logs/HEAD:e3e8c3f2a0fe4575d565013292023584688036e3 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548704539 -0500 checkout: moving from DC-60-dbt-initial-setup to master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704552 -0500 pull rayne DC-18-extract-main-executor: Fast-forward +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704562 -0500 checkout: moving from master to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704579 -0500 checkout: moving from master to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704587 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704609 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704638 -0500 reset: moving to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704671 -0500 checkout: moving from master to rayne/DC-18-extract-main-executor +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548704721 -0500 checkout: moving from 115661485519d8c390fd07fd9e2a34e6baefc060 to official/master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704821 -0500 checkout: moving from 1ed76759d266c4f45eeb8b8d40777a800f254df5 to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704827 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704834 -0500 checkout: moving from master to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704876 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704883 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704933 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704977 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548705114 -0500 checkout: moving from master to 1ed76759d266c4f45eeb8b8d40777a800f254df5 +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548705127 -0500 checkout: moving from 1ed76759d266c4f45eeb8b8d40777a800f254df5 to throw_away +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705135 -0500 checkout: moving from throw_away to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705188 -0500 reset: moving to HEAD +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548705228 -0500 checkout: moving from master to origin/master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705327 -0500 checkout: moving from 1ed76759d266c4f45eeb8b8d40777a800f254df5 to master +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705685 -0500 checkout: moving from master to throw_away +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548705746 -0500 checkout: moving from throw_away to official/master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705771 -0500 checkout: moving from 1ed76759d266c4f45eeb8b8d40777a800f254df5 to throw_away +./.git/logs/HEAD:115661485519d8c390fd07fd9e2a34e6baefc060 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548705790 -0500 checkout: moving from throw_away to origin/master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548706327 -0500 reset: moving to HEAD~2 +./.git/logs/HEAD:ea9a3169cd8152de92a1acf7b02386d97a4be895 ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548706349 -0500 checkout: moving from ea9a3169cd8152de92a1acf7b02386d97a4be895 to master +./.git/logs/HEAD:ea9a3169cd8152de92a1acf7b02386d97a4be895 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548706358 -0500 pull origin master: Fast-forward +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 e3e8c3f2a0fe4575d565013292023584688036e3 Ethan Knox 1548706785 -0500 checkout: moving from master to DC-60-dbt-initial-setup +./.git/logs/HEAD:e3e8c3f2a0fe4575d565013292023584688036e3 40d39a5568ad09c005cd9ff62bb7e8809d49ce65 Ethan Knox 1548722713 -0500 commit: docs revisited +./.git/logs/HEAD:40d39a5568ad09c005cd9ff62bb7e8809d49ce65 d3e5b09227e7aaf5cf65a8532014bde24a24f62c Ethan Knox 1548722960 -0500 commit (amend): DC-60 #groomed #comment initial repo setup +./.git/logs/HEAD:d3e5b09227e7aaf5cf65a8532014bde24a24f62c 06179e7eefc87e973c9dd49c5d1976aa1c70202f Ethan Knox 1548724289 -0500 commit: DC-60 #groomed +./.git/logs/HEAD:06179e7eefc87e973c9dd49c5d1976aa1c70202f 2d31030301c45e06cb3fe045d01b399b304b90e5 Ethan Knox 1548724327 -0500 commit: DC-60 #Groomed +./.git/logs/HEAD:2d31030301c45e06cb3fe045d01b399b304b90e5 c28e39b1f1edf24372edb2b124033e7314209d07 Ethan Knox 1548724432 -0500 commit: DC-60 #groomed #comment pointed ticket +./.git/logs/HEAD:c28e39b1f1edf24372edb2b124033e7314209d07 580e09ee3ba0a32c861e255c5f7d9ad110119106 Ethan Knox 1548771046 -0500 commit: dbt working console +./.git/logs/HEAD:580e09ee3ba0a32c861e255c5f7d9ad110119106 db38de23635fd902c8624fd8c2fb9a9d6f98bf72 Ethan Knox 1548776777 -0500 commit: migrating profiles +./.git/logs/HEAD:db38de23635fd902c8624fd8c2fb9a9d6f98bf72 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776784 -0500 checkout: moving from DC-60-dbt-initial-setup to master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 3f610abac2a461590d5ecfca88300ad03d9db464 Ethan Knox 1548776894 -0500 checkout: moving from master to official/DC-18-extract-main-executor +./.git/logs/HEAD:3f610abac2a461590d5ecfca88300ad03d9db464 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776945 -0500 checkout: moving from 3f610abac2a461590d5ecfca88300ad03d9db464 to master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776977 -0500 checkout: moving from master to DC-18-extract-main-executort +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 0000000000000000000000000000000000000000 Ethan Knox 1548776993 -0500 Branch: renamed refs/heads/DC-18-extract-main-executort to refs/heads/DC-18-extract-main-executor +./.git/logs/HEAD:0000000000000000000000000000000000000000 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776993 -0500 Branch: renamed refs/heads/DC-18-extract-main-executort to refs/heads/DC-18-extract-main-executor +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 3f610abac2a461590d5ecfca88300ad03d9db464 Ethan Knox 1548777071 -0500 pull official DC-18-extract-main-executor: Fast-forward +./.git/logs/HEAD:3f610abac2a461590d5ecfca88300ad03d9db464 7b66a4c33dc278dbe27260493cd16532c77ff16d Ethan Knox 1548785171 -0500 commit: added secret_type_of to extract_configuration +./.git/logs/HEAD:7b66a4c33dc278dbe27260493cd16532c77ff16d 19bb9ba2b78848dfba0701edb7b2c2e175829970 Ethan Knox 1548785637 -0500 pull official DC-18-extract-main-executor: Fast-forward +./.git/logs/HEAD:19bb9ba2b78848dfba0701edb7b2c2e175829970 c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 Ethan Knox 1548787145 -0500 commit: removed hardcoded test vals in contract +./.git/logs/HEAD:c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548787164 -0500 checkout: moving from DC-18-extract-main-executor to master +./.git/logs/HEAD:1ed76759d266c4f45eeb8b8d40777a800f254df5 c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 Ethan Knox 1548787242 -0500 checkout: moving from master to DC-18-extract-main-executor +./.git/logs/refs/heads/DC-39-contract-getter-setter:0000000000000000000000000000000000000000 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548426606 -0500 branch: Created from HEAD +./.git/logs/refs/heads/DC-39-contract-getter-setter:557c1ad49e7f62dbd774e9a79842c3a8249189da a8dee17dd2708596a8c67ac7825ed40ee54279d8 Ethan Knox 1548447398 -0500 commit: invalid test +./.git/logs/refs/heads/DC-39-contract-getter-setter:a8dee17dd2708596a8c67ac7825ed40ee54279d8 d4641fc755b364d2fc9a8a91ca40e2585d48755d Ethan Knox 1548447437 -0500 commit: autopep8 +./.git/logs/refs/heads/DC-39-contract-getter-setter:d4641fc755b364d2fc9a8a91ca40e2585d48755d 2260bc6295cf5adfe0d6644b967672a516a81298 Ethan Knox 1548448339 -0500 commit: docs +./.git/logs/refs/heads/DC-39-contract-getter-setter:2260bc6295cf5adfe0d6644b967672a516a81298 44c17d4606362784c4e45f99f579f429a5117892 Ethan Knox 1548448825 -0500 commit: meta test +./.git/logs/refs/heads/DC-60-dbt-initial-setup:0000000000000000000000000000000000000000 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548696825 -0500 branch: Created from HEAD +./.git/logs/refs/heads/DC-60-dbt-initial-setup:1ed76759d266c4f45eeb8b8d40777a800f254df5 6d6bfffd2b119e102a431ecabd351622e44fd6d3 Ethan Knox 1548702392 -0500 commit: docs +./.git/logs/refs/heads/DC-60-dbt-initial-setup:6d6bfffd2b119e102a431ecabd351622e44fd6d3 e3e8c3f2a0fe4575d565013292023584688036e3 Ethan Knox 1548704534 -0500 commit: svg support +./.git/logs/refs/heads/DC-60-dbt-initial-setup:e3e8c3f2a0fe4575d565013292023584688036e3 40d39a5568ad09c005cd9ff62bb7e8809d49ce65 Ethan Knox 1548722713 -0500 commit: docs revisited +./.git/logs/refs/heads/DC-60-dbt-initial-setup:40d39a5568ad09c005cd9ff62bb7e8809d49ce65 d3e5b09227e7aaf5cf65a8532014bde24a24f62c Ethan Knox 1548722960 -0500 commit (amend): DC-60 #groomed #comment initial repo setup +./.git/logs/refs/heads/DC-60-dbt-initial-setup:d3e5b09227e7aaf5cf65a8532014bde24a24f62c 06179e7eefc87e973c9dd49c5d1976aa1c70202f Ethan Knox 1548724289 -0500 commit: DC-60 #groomed +./.git/logs/refs/heads/DC-60-dbt-initial-setup:06179e7eefc87e973c9dd49c5d1976aa1c70202f 2d31030301c45e06cb3fe045d01b399b304b90e5 Ethan Knox 1548724327 -0500 commit: DC-60 #Groomed +./.git/logs/refs/heads/DC-60-dbt-initial-setup:2d31030301c45e06cb3fe045d01b399b304b90e5 c28e39b1f1edf24372edb2b124033e7314209d07 Ethan Knox 1548724432 -0500 commit: DC-60 #groomed #comment pointed ticket +./.git/logs/refs/heads/DC-60-dbt-initial-setup:c28e39b1f1edf24372edb2b124033e7314209d07 580e09ee3ba0a32c861e255c5f7d9ad110119106 Ethan Knox 1548771046 -0500 commit: dbt working console +./.git/logs/refs/heads/DC-60-dbt-initial-setup:580e09ee3ba0a32c861e255c5f7d9ad110119106 db38de23635fd902c8624fd8c2fb9a9d6f98bf72 Ethan Knox 1548776777 -0500 commit: migrating profiles +./.git/logs/refs/heads/master:0000000000000000000000000000000000000000 ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548706349 -0500 branch: Created from HEAD +./.git/logs/refs/heads/master:ea9a3169cd8152de92a1acf7b02386d97a4be895 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548706358 -0500 pull origin master: Fast-forward +./.git/logs/refs/heads/DC-18-extract-main-executor:0000000000000000000000000000000000000000 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776977 -0500 branch: Created from HEAD +./.git/logs/refs/heads/DC-18-extract-main-executor:1ed76759d266c4f45eeb8b8d40777a800f254df5 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548776993 -0500 Branch: renamed refs/heads/DC-18-extract-main-executort to refs/heads/DC-18-extract-main-executor +./.git/logs/refs/heads/DC-18-extract-main-executor:1ed76759d266c4f45eeb8b8d40777a800f254df5 3f610abac2a461590d5ecfca88300ad03d9db464 Ethan Knox 1548777071 -0500 pull official DC-18-extract-main-executor: Fast-forward +./.git/logs/refs/heads/DC-18-extract-main-executor:3f610abac2a461590d5ecfca88300ad03d9db464 7b66a4c33dc278dbe27260493cd16532c77ff16d Ethan Knox 1548785171 -0500 commit: added secret_type_of to extract_configuration +./.git/logs/refs/heads/DC-18-extract-main-executor:7b66a4c33dc278dbe27260493cd16532c77ff16d 19bb9ba2b78848dfba0701edb7b2c2e175829970 Ethan Knox 1548785637 -0500 pull official DC-18-extract-main-executor: Fast-forward +./.git/logs/refs/heads/DC-18-extract-main-executor:19bb9ba2b78848dfba0701edb7b2c2e175829970 c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 Ethan Knox 1548787145 -0500 commit: removed hardcoded test vals in contract +./.git/logs/refs/remotes/nate/DC-111-add_batch_job_config_to_cli:0000000000000000000000000000000000000000 b84eaed4a98f8f8ebc46bad463ec3e74e0810101 Ethan Knox 1548692547 -0500 fetch nate --prune: storing head +./.git/logs/refs/remotes/nate/DC-111-register_docker_image:0000000000000000000000000000000000000000 6fead2c53b081debfd8af6a9ca3b3b66ae0208c4 Ethan Knox 1548692547 -0500 fetch nate --prune: storing head +./.git/logs/refs/remotes/nate/DC-111-launch_job_on_batch:0000000000000000000000000000000000000000 0ac9165baab129a2a5d853a70561ed5ca9be0ba8 Ethan Knox 1548692547 -0500 fetch nate --prune: storing head +./.git/logs/refs/remotes/nate/master:0000000000000000000000000000000000000000 6e3c6b607d2b56403716da0769350d3788d37c96 Ethan Knox 1547665312 -0500 fetch nate: storing head +./.git/logs/refs/remotes/nate/master:6e3c6b607d2b56403716da0769350d3788d37c96 70a5efdc17b7ff5e10464859aa1f1583c8f2a827 Ethan Knox 1548692547 -0500 fetch nate --prune: fast-forward +./.git/logs/refs/remotes/nate/DC-114-lauch_notebook_job:0000000000000000000000000000000000000000 931c4e8e8ae551da1877970532af0cbb08b40f15 Ethan Knox 1548692547 -0500 fetch nate --prune: storing head +./.git/logs/refs/remotes/rayne/master:0000000000000000000000000000000000000000 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1548704544 -0500 fetch rayne: storing head +./.git/logs/refs/remotes/rayne/DC-18-extract-main-executor:0000000000000000000000000000000000000000 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548704544 -0500 fetch rayne: storing head +./.git/logs/refs/remotes/official/master:0000000000000000000000000000000000000000 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547665171 -0500 pull official master: storing head +./.git/logs/refs/remotes/official/master:c9bfd40daabdf7df72633fdb265987baed46ce5c a65b20c41d28324fe210f4afc198b48cfdeb570c Ethan Knox 1547731492 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:a65b20c41d28324fe210f4afc198b48cfdeb570c eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1547833825 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:eb1bb9427d9917063f7532b7a864d554feb5292e 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548084858 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:2034b47676284a340fc29f22bb4ab60487cb50dd cd720a2e6881b364bc510d916f89fe7dff6c0f48 Ethan Knox 1548268919 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:cd720a2e6881b364bc510d916f89fe7dff6c0f48 70a5efdc17b7ff5e10464859aa1f1583c8f2a827 Ethan Knox 1548274697 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:70a5efdc17b7ff5e10464859aa1f1583c8f2a827 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548336532 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:f2deec00885baf1fc5332668faacc51581888bd1 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548367571 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:557c1ad49e7f62dbd774e9a79842c3a8249189da ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548449177 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:ea9a3169cd8152de92a1acf7b02386d97a4be895 24756276bdca657032989db442c8b6a5135a343e Ethan Knox 1548454526 -0500 pull official master: fast-forward +./.git/logs/refs/remotes/official/master:24756276bdca657032989db442c8b6a5135a343e 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548692467 -0500 fetch official: fast-forward +./.git/logs/refs/remotes/official/DC-18-extract-main-executor:0000000000000000000000000000000000000000 115661485519d8c390fd07fd9e2a34e6baefc060 Ethan Knox 1548705702 -0500 fetch official: storing head +./.git/logs/refs/remotes/official/DC-18-extract-main-executor:115661485519d8c390fd07fd9e2a34e6baefc060 3f610abac2a461590d5ecfca88300ad03d9db464 Ethan Knox 1548776864 -0500 fetch official: fast-forward +./.git/logs/refs/remotes/official/DC-18-extract-main-executor:3f610abac2a461590d5ecfca88300ad03d9db464 7b66a4c33dc278dbe27260493cd16532c77ff16d Ethan Knox 1548785188 -0500 update by push +./.git/logs/refs/remotes/official/DC-18-extract-main-executor:7b66a4c33dc278dbe27260493cd16532c77ff16d 19bb9ba2b78848dfba0701edb7b2c2e175829970 Ethan Knox 1548785637 -0500 pull official DC-18-extract-main-executor: fast-forward +./.git/logs/refs/remotes/official/DC-18-extract-main-executor:19bb9ba2b78848dfba0701edb7b2c2e175829970 c440c6bf36a6ee9f4a9332e6c9788c1e1a740b58 Ethan Knox 1548787158 -0500 update by push +./.git/logs/refs/remotes/origin/DC-39-contract-getter-setter:0000000000000000000000000000000000000000 2260bc6295cf5adfe0d6644b967672a516a81298 Ethan Knox 1548448349 -0500 update by push +./.git/logs/refs/remotes/origin/DC-39-contract-getter-setter:2260bc6295cf5adfe0d6644b967672a516a81298 44c17d4606362784c4e45f99f579f429a5117892 Ethan Knox 1548448832 -0500 update by push +./.git/logs/refs/remotes/origin/HEAD:0000000000000000000000000000000000000000 c9bfd40daabdf7df72633fdb265987baed46ce5c Ethan Knox 1547606669 -0500 clone: from git@github.com:norton120/core.git +./.git/logs/refs/remotes/origin/DC-60-dbt-initial-setup:0000000000000000000000000000000000000000 d3e5b09227e7aaf5cf65a8532014bde24a24f62c Ethan Knox 1548722981 -0500 update by push +./.git/logs/refs/remotes/origin/DC-60-dbt-initial-setup:d3e5b09227e7aaf5cf65a8532014bde24a24f62c 06179e7eefc87e973c9dd49c5d1976aa1c70202f Ethan Knox 1548724294 -0500 update by push +./.git/logs/refs/remotes/origin/DC-60-dbt-initial-setup:06179e7eefc87e973c9dd49c5d1976aa1c70202f 2d31030301c45e06cb3fe045d01b399b304b90e5 Ethan Knox 1548724331 -0500 update by push +./.git/logs/refs/remotes/origin/DC-60-dbt-initial-setup:2d31030301c45e06cb3fe045d01b399b304b90e5 c28e39b1f1edf24372edb2b124033e7314209d07 Ethan Knox 1548724436 -0500 update by push +./.git/logs/refs/remotes/origin/master:c9bfd40daabdf7df72633fdb265987baed46ce5c a65b20c41d28324fe210f4afc198b48cfdeb570c Ethan Knox 1547731502 -0500 update by push +./.git/logs/refs/remotes/origin/master:a65b20c41d28324fe210f4afc198b48cfdeb570c eb1bb9427d9917063f7532b7a864d554feb5292e Ethan Knox 1547833836 -0500 update by push +./.git/logs/refs/remotes/origin/master:eb1bb9427d9917063f7532b7a864d554feb5292e 2034b47676284a340fc29f22bb4ab60487cb50dd Ethan Knox 1548084885 -0500 update by push +./.git/logs/refs/remotes/origin/master:2034b47676284a340fc29f22bb4ab60487cb50dd 70a5efdc17b7ff5e10464859aa1f1583c8f2a827 Ethan Knox 1548275972 -0500 update by push +./.git/logs/refs/remotes/origin/master:70a5efdc17b7ff5e10464859aa1f1583c8f2a827 f2deec00885baf1fc5332668faacc51581888bd1 Ethan Knox 1548336541 -0500 update by push +./.git/logs/refs/remotes/origin/master:f2deec00885baf1fc5332668faacc51581888bd1 557c1ad49e7f62dbd774e9a79842c3a8249189da Ethan Knox 1548367579 -0500 update by push +./.git/logs/refs/remotes/origin/master:557c1ad49e7f62dbd774e9a79842c3a8249189da ea9a3169cd8152de92a1acf7b02386d97a4be895 Ethan Knox 1548449184 -0500 update by push +./.git/logs/refs/remotes/origin/master:ea9a3169cd8152de92a1acf7b02386d97a4be895 1ed76759d266c4f45eeb8b8d40777a800f254df5 Ethan Knox 1548692484 -0500 update by push +./.git/logs/refs/remotes/origin/DC-131-constants:0000000000000000000000000000000000000000 e5cd40e9e96720e9cf31d44d8e4ea019faad5103 Ethan Knox 1548451440 -0500 update by push +./.git/logs/refs/remotes/origin/DC-131-constants:e5cd40e9e96720e9cf31d44d8e4ea019faad5103 36c012be1a1f3fdd7e9bbfa3a11c7e38e0f7d32b Ethan Knox 1548454389 -0500 update by push +./.git/logs/refs/remotes/origin/DC-131-constants:36c012be1a1f3fdd7e9bbfa3a11c7e38e0f7d32b f5f91da71588e96d66892a024c65afbc9bd684db Ethan Knox 1548454561 -0500 update by push +./.git/hooks/commit-msg.sample:# that has the commit message. The hook should exit with non-zero +./.git/hooks/commit-msg.sample:# To enable this hook, rename this file to "commit-msg". +./.git/hooks/commit-msg.sample:# Uncomment the below to add a Signed-off-by line to the message. +./.git/hooks/commit-msg.sample:# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +./.git/hooks/commit-msg.sample:# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +./.git/hooks/commit-msg.sample:# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" +./.git/hooks/commit-msg.sample:# This example catches duplicate Signed-off-by lines. +./.git/hooks/commit-msg.sample:test "" = "$(grep '^Signed-off-by: ' "$1" | +./.git/hooks/commit-msg.sample: sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { +./.git/hooks/commit-msg.sample: echo >&2 Duplicate Signed-off-by lines. +./.git/hooks/pre-rebase.sample:# The "pre-rebase" hook is run just before "git rebase" starts doing +./.git/hooks/pre-rebase.sample:# non-zero status. +./.git/hooks/pre-rebase.sample:# $1 -- the upstream the series was forked from. +./.git/hooks/pre-rebase.sample:# $2 -- the branch being rebased (or empty when rebasing the current branch). +./.git/hooks/pre-rebase.sample: topic=`git symbolic-ref HEAD` || +./.git/hooks/pre-rebase.sample:git show-ref -q "$topic" || { +./.git/hooks/pre-rebase.sample:not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +./.git/hooks/pre-rebase.sample:if test -z "$not_in_master" +./.git/hooks/pre-rebase.sample:only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +./.git/hooks/pre-rebase.sample:only_next_2=`git rev-list ^master ${publish} | sort` +./.git/hooks/pre-rebase.sample: not_in_topic=`git rev-list "^$topic" master` +./.git/hooks/pre-rebase.sample: if test -z "$not_in_topic" +./.git/hooks/pre-rebase.sample: not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` +./.git/hooks/pre-rebase.sample: /usr/bin/perl -e ' +./.git/hooks/pre-rebase.sample: /^([0-9a-f]+) /; +./.git/hooks/pre-rebase.sample: /^([0-9a-f]+) (.*)$/; +./.git/hooks/pre-rebase.sample: if (!exists $not_in_next{$elem->[0]}) { +./.git/hooks/pre-rebase.sample: print STDERR " $elem->[1]\n"; +./.git/hooks/pre-rebase.sample: build on top of it -- other people may already want to +./.git/hooks/pre-rebase.sample: o---o---o---o---o---o---o---o---o---o "next" +./.git/hooks/pre-rebase.sample: / a---a---b A / / +./.git/hooks/pre-rebase.sample: / / c---c---c---c B / +./.git/hooks/pre-rebase.sample: / / / b---b C \ / +./.git/hooks/pre-rebase.sample: ---o---o---o---o---o---o---o---o---o---o---o "master" +./.git/hooks/pre-rebase.sample: git rev-list ^master ^topic next +./.git/hooks/pre-rebase.sample: git rev-list ^master next +./.git/hooks/pre-rebase.sample: git rev-list master..topic +./.git/hooks/pre-commit.sample:# exit with non-zero status after issuing an appropriate message if +./.git/hooks/pre-commit.sample:# To enable this hook, rename this file to "pre-commit". +./.git/hooks/pre-commit.sample:if git rev-parse --verify HEAD >/dev/null 2>&1 +./.git/hooks/pre-commit.sample:# If you want to allow non-ASCII filenames set this variable to true. +./.git/hooks/pre-commit.sample:allownonascii=$(git config --bool hooks.allownonascii) +./.git/hooks/pre-commit.sample:# Cross platform projects tend to avoid non-ASCII filenames; prevent +./.git/hooks/pre-commit.sample: test $(git diff --cached --name-only --diff-filter=A -z $against | +./.git/hooks/pre-commit.sample: LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +./.git/hooks/pre-commit.sample:Error: Attempt to add a non-ASCII file name. +./.git/hooks/pre-commit.sample:exec git diff-index --check --cached $against -- +./.git/hooks/applypatch-msg.sample:# applypatch from an e-mail message. +./.git/hooks/applypatch-msg.sample:# The hook should exit with non-zero status after issuing an +./.git/hooks/applypatch-msg.sample:# To enable this hook, rename this file to "applypatch-msg". +./.git/hooks/applypatch-msg.sample:. git-sh-setup +./.git/hooks/applypatch-msg.sample:commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +./.git/hooks/applypatch-msg.sample:test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +./.git/hooks/fsmonitor-watchman.sample:# To enable this hook, rename this file to "query-watchman" and set +./.git/hooks/fsmonitor-watchman.sample:# 'git config core.fsmonitor .git/hooks/query-watchman' +./.git/hooks/fsmonitor-watchman.sample: die "Unsupported query-fsmonitor hook version '$version'.\n" . +./.git/hooks/fsmonitor-watchman.sample: my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') +./.git/hooks/fsmonitor-watchman.sample: my $o = $json_pkg->new->utf8->decode($response); +./.git/hooks/fsmonitor-watchman.sample: if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { +./.git/hooks/fsmonitor-watchman.sample: $retry--; +./.git/hooks/fsmonitor-watchman.sample: die "Watchman: $o->{error}.\n" . +./.git/hooks/fsmonitor-watchman.sample: "Falling back to scanning...\n" if $o->{error}; +./.git/hooks/fsmonitor-watchman.sample: print @{$o->{files}}; +./.git/hooks/pre-receive.sample:# To enable this hook, rename this file to "pre-receive". +./.git/hooks/pre-receive.sample:if test -n "$GIT_PUSH_OPTION_COUNT" +./.git/hooks/pre-receive.sample: while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" +./.git/hooks/pre-receive.sample: echo "echo from the pre-receive-hook: ${value#*=}" >&2 +./.git/hooks/prepare-commit-msg.sample:# message file. If the hook fails with a non-zero status, +./.git/hooks/prepare-commit-msg.sample:# To enable this hook, rename this file to "prepare-commit-msg". +./.git/hooks/prepare-commit-msg.sample:# The second includes the output of "git diff --name-status -r" +./.git/hooks/prepare-commit-msg.sample:# commented because it doesn't cope with --amend or with squashed +./.git/hooks/prepare-commit-msg.sample:# The third example adds a Signed-off-by line to the message, that can +./.git/hooks/prepare-commit-msg.sample:/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" +./.git/hooks/prepare-commit-msg.sample:# /usr/bin/perl -i.bak -pe ' +./.git/hooks/prepare-commit-msg.sample:# print "\n" . `git diff --cached --name-status -r` +./.git/hooks/prepare-commit-msg.sample:# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +./.git/hooks/prepare-commit-msg.sample:# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" +./.git/hooks/prepare-commit-msg.sample:# if test -z "$COMMIT_SOURCE" +./.git/hooks/prepare-commit-msg.sample:# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" +./.git/hooks/post-update.sample:# To enable this hook, rename this file to "post-update". +./.git/hooks/post-update.sample:exec git update-server-info +./.git/hooks/pre-applypatch.sample:# by applypatch from an e-mail message. +./.git/hooks/pre-applypatch.sample:# The hook should exit with non-zero status after issuing an +./.git/hooks/pre-applypatch.sample:# To enable this hook, rename this file to "pre-applypatch". +./.git/hooks/pre-applypatch.sample:. git-sh-setup +./.git/hooks/pre-applypatch.sample:precommit="$(git rev-parse --git-path hooks/pre-commit)" +./.git/hooks/pre-applypatch.sample:test -x "$precommit" && exec "$precommit" ${1+"$@"} +./.git/hooks/pre-push.sample:# pushed. If this script exits with a non-zero status nothing will be pushed. +./.git/hooks/pre-push.sample:# $1 -- Name of the remote to which the push is being done +./.git/hooks/pre-push.sample:# $2 -- URL to which the push is being done +./.git/hooks/pre-push.sample: commit=`git rev-list -n 1 --grep '^WIP' "$range"` +./.git/hooks/pre-push.sample: if [ -n "$commit" ] +./.git/hooks/update.sample:# Called by "git receive-pack" with arguments: refname sha1-old sha1-new +./.git/hooks/update.sample:# ------ +./.git/hooks/update.sample:# --- Command line +./.git/hooks/update.sample:# --- Safety check +./.git/hooks/update.sample:if [ -z "$GIT_DIR" ]; then +./.git/hooks/update.sample:if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then +./.git/hooks/update.sample:# --- Config +./.git/hooks/update.sample:allowunannotated=$(git config --bool hooks.allowunannotated) +./.git/hooks/update.sample:allowdeletebranch=$(git config --bool hooks.allowdeletebranch) +./.git/hooks/update.sample:denycreatebranch=$(git config --bool hooks.denycreatebranch) +./.git/hooks/update.sample:allowdeletetag=$(git config --bool hooks.allowdeletetag) +./.git/hooks/update.sample:allowmodifytag=$(git config --bool hooks.allowmodifytag) +./.git/hooks/update.sample:projectdesc=$(sed -e '1q' "$GIT_DIR/description") +./.git/hooks/update.sample:# --- Check types +./.git/hooks/update.sample: newrev_type=$(git cat-file -t $newrev) +./.git/hooks/update.sample: # un-annotated tag +./.git/hooks/update.sample: echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2 +./.git/hooks/update.sample: echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 +./.git/hooks/update.sample: if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 +./.git/hooks/update.sample: if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then +./.git/hooks/update.sample:# --- Finished +Binary file ./.git/index matches +./.git/packed-refs:# pack-refs with: peeled fully-peeled sorted +./.git/FETCH_HEAD:19bb9ba2b78848dfba0701edb7b2c2e175829970 branch 'DC-18-extract-main-executor' of github.com:IntegriChain1/core +./dockerfiles/notebook.dockerfile:FROM jupyter/pyspark-notebook +./dockerfiles/notebook.dockerfile:RUN wget -q https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_CONNECTOR_VERSION}/mysql-connector-java-${MYSQL_CONNECTOR_VERSION}.jar -P $SPARK_HOME/jars +./dockerfiles/notebook.dockerfile:RUN wget -q http://central.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION_FULL}/hadoop-aws-${HADOOP_VERSION_FULL}.jar -P $SPARK_HOME/jars +./dockerfiles/notebook.dockerfile:RUN wget -q http://central.maven.org/maven2/com/amazonaws/aws-java-sdk/${AWS_JAVA_SDK_VERSION}/aws-java-sdk-${AWS_JAVA_SDK_VERSION}.jar -P $SPARK_HOME/jars +./dockerfiles/notebook.dockerfile:RUN pip install -r requirements.txt +./dockerfiles/notebook.dockerfile:RUN fix-permissions $CONDA_DIR +./dockerfiles/notebook.dockerfile:ENTRYPOINT ["/usr/local/bin/start-notebook.sh"] +./dockerfiles/python3-7.dockerfile:RUN apt-get update +./dockerfiles/python3-7.dockerfile:RUN apt-get install -y \ +./dockerfiles/python3-7.dockerfile: apt-transport-https \ +./dockerfiles/python3-7.dockerfile: ca-certificates \ +./dockerfiles/python3-7.dockerfile: software-properties-common +./dockerfiles/python3-7.dockerfile:RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +./dockerfiles/python3-7.dockerfile:RUN apt-key fingerprint 0EBFCD88 +./dockerfiles/python3-7.dockerfile:RUN add-apt-repository \ +./dockerfiles/python3-7.dockerfile: $(lsb_release -cs) \ +./dockerfiles/python3-7.dockerfile:RUN apt-get update +./dockerfiles/python3-7.dockerfile:RUN apt-get install -y docker-ce +./dockerfiles/python3-7.dockerfile:RUN pip install --no-cache-dir -r requirements.txt +./dockerfiles/core.dockerfile:RUN apt-get update +./dockerfiles/core.dockerfile:RUN apt-get install -y \ +./dockerfiles/core.dockerfile: apt-transport-https \ +./dockerfiles/core.dockerfile: ca-certificates \ +./dockerfiles/core.dockerfile: software-properties-common +./dockerfiles/core.dockerfile:RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +./dockerfiles/core.dockerfile:RUN apt-key fingerprint 0EBFCD88 +./dockerfiles/core.dockerfile:RUN add-apt-repository \ +./dockerfiles/core.dockerfile: $(lsb_release -cs) \ +./dockerfiles/core.dockerfile:RUN apt-get update +./dockerfiles/core.dockerfile:RUN apt-get install -y docker-ce +./dockerfiles/core.dockerfile:RUN pip install --no-cache-dir -r requirements.txt diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py new file mode 100644 index 00000000..fc5c1595 --- /dev/null +++ b/tests/unit/test_extract.py @@ -0,0 +1,63 @@ +import pytest +from core.helpers.configuration_mocker import ConfigurationMocker as CMock +import core.models.configuation as C +import core.contract as contract +from core.transforms.shared.raw.extract import ExtractTransform +import boto3 +import moto +import time + +''' +@moto.mock_s3 +def setup_configs(): + ENV = "dev" + mock = CMock() + mock.generate_mocks() + transform = C.Transformation() + session = mock.get_session() + t_target = session.query(transform).one() + return t_target + +@moto.mock_s3 +def setup_contract(): + ct = Contract( env=ENV, + parent = , + child = , + branch = , + state = , + ) + return ct + + + + ex = ExtractTransform(ENV, t_target, + +''' +@moto.mock_s3 +def s3_setup(): + client = boto3.client('s3') + time = time.time() + file_name = 'sandwiches12345.txt' + BINARY_DATA = b'Gobias some coffee!' + CONTRACT_PATH = f'master/bluth/cornballer/raw/{file_name}' + client.create_bucket(Bucket= DEV_BUCKET) + client.put_object(Body=BINARY_DATA, Bucket=BUCKET, Key= CONTRACT_PATH, ExtraArgs={"Metadata":{"source_modified_time": str(time - 10000000)}}) + return (file_name, time,) + + +def test_push_to_s3_new_file(): + params = s3_setup() + + +def test_push_to_s3_updated_file(): + pass + +def test_push_to_s3_not_if_older(): + pass + +def test_file_needs_update_needs_update(): + pass + +def test_file_needs_update_doesnt_need_update(): + pass + diff --git a/transforms/__init__.py b/transforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/transforms/shared/__init__.py b/transforms/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/transforms/shared/raw/__init__.py b/transforms/shared/raw/__init__.py new file mode 100644 index 00000000..e69de29b From 73d4273accfac0d350daa113bfb397f474f48f9a Mon Sep 17 00:00:00 2001 From: "Rayne N. Schaffer" Date: Tue, 29 Jan 2019 15:29:23 -0500 Subject: [PATCH 7/9] updating extract class to updates --- core/transforms/shared/raw/extract.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/transforms/shared/raw/extract.py b/core/transforms/shared/raw/extract.py index 11277de2..34a353d8 100644 --- a/core/transforms/shared/raw/extract.py +++ b/core/transforms/shared/raw/extract.py @@ -26,7 +26,7 @@ def run(self): # Fetch secret from secret contract # TODO: Currently configs made for FTP only, FTP type passed in directly - source_secret = secret.Secret(name=config.secret_name,env=self.env,type_of="FTP",mode="write") + source_secret = secret.Secret(name=config.secret_name,env=self.env,type_of=config.secret_type_of,mode="write") # Get files from remote and start pushing to s3 with tempfile.TemporaryDirectory() as tmp_dir: @@ -34,7 +34,7 @@ def run(self): self.push_to_s3(tmp_dir, self.output_contract) - def push_to_s3(self, tmp_dir: str, output_contract: core.contract)-> None: + def push_to_s3(self, tmp_dir: str, output_contract: contract)-> None: """ For a local file dir, push the file to s3 if it is newer or does not exist.""" # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints @@ -42,14 +42,15 @@ def push_to_s3(self, tmp_dir: str, output_contract: core.contract)-> None: local_file_path = os.path.join(tmp_dir,local_file) local_file_modified_time = os.stat(os.path.join(tmp_dir,local_file)).st_mtime - if (self._file_needs_update(local_file_path,local_file_modified_time)): + if (self._file_needs_update(output_contract=output_contract, + local_file_path=local_file_path, + local_file_modified_time=local_file_modified_time + )): output_contract.publish_raw_file(local_file_path) - def _file_needs_update(self,local_file_path,local_file_modified_time): + def _file_needs_update(self,output_contract: contract,local_file_path: str,local_file_modified_time: str)-> None: # Check if file needs to be pushed # File is only considered to need to be pushed if it does not exist or has been modified since last push - output_contract = self.output_contract - try: s3_last_modified = output_contract.get_raw_file_metadata(local_file_path)['Metadata']['source_modified_time'] if (float(s3_last_modified) < float(local_file_modified_time)): From 55f4c00f61b45ece4078abffc1a64162a2d89a1a Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 29 Jan 2019 17:13:44 -0500 Subject: [PATCH 8/9] test #1! --- core/transforms/shared/raw/extract.py | 42 +++++++++++++++++--- tests/unit/test_extract.py | 55 ++++++++++----------------- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/core/transforms/shared/raw/extract.py b/core/transforms/shared/raw/extract.py index 11277de2..eb23d498 100644 --- a/core/transforms/shared/raw/extract.py +++ b/core/transforms/shared/raw/extract.py @@ -6,16 +6,39 @@ import tempfile class ExtractTransform(): - def __init__(self, env:str, transform: core.models.configuration.Transformation, output_contract: core.contract) -> None: + + def __init__(self, **kwargs) -> None: """ performs the extraction to a given output contract. - ARGS: - - env one of "dev", "prod", "UAT" + valid kwargs: + - env one of "dev", "prod", "uat" - transform a configuration contract instance - output_contract a contract instance """ - self.env = env - self.output_contract = output_contract + self.REQUIRED_PARAMS = ('env','output_contract','transform') + + for attr in self.REQUIRED_PARAMS: + self.__dict__[attr] = None + + for attr in self.REQUIRED_PARAMS: + if attr in kwargs: + setter = getattr(self, 'set_' + attr) + setter(kwargs[attr]) + + + def set_env(self,env:str)->None: + if env in ('dev','prod','uat'): + self.env = env + else: + raise ValueError(f'{env} is not a valid environment') + + def set_transform(self, transform: configuration.Transformation) -> None: self.transform = transform + + def set_output_contract(self, output_contract: contract) -> None: + self.output_contract = output_contract + + + def run(self): for config in self.transform.extract_configurations: @@ -34,8 +57,9 @@ def run(self): self.push_to_s3(tmp_dir, self.output_contract) - def push_to_s3(self, tmp_dir: str, output_contract: core.contract)-> None: + def push_to_s3(self, tmp_dir: str, output_contract: contract)-> None: """ For a local file dir, push the file to s3 if it is newer or does not exist.""" + self._validate_required_params() # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints for local_file in os.listdir(f"{tmp_dir}"): @@ -58,3 +82,9 @@ def _file_needs_update(self,local_file_path,local_file_modified_time): return False except: return True + + def _validate_required_params(self) -> bool: + ''' Checks that all required params are set ''' + for param in self.REQUIRED_PARAMS: + if param not in self.__dict__.keys(): + raise ValueError(f'{param} is a required value not set for ExtractTransform.') diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index fc5c1595..6dd9c0be 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -1,55 +1,40 @@ import pytest +from core.constants import DEV_BUCKET from core.helpers.configuration_mocker import ConfigurationMocker as CMock -import core.models.configuation as C +import core.models.configuration as C import core.contract as contract from core.transforms.shared.raw.extract import ExtractTransform import boto3 import moto import time +import tempfile +import os -''' -@moto.mock_s3 -def setup_configs(): - ENV = "dev" - mock = CMock() - mock.generate_mocks() - transform = C.Transformation() - session = mock.get_session() - t_target = session.query(transform).one() - return t_target - -@moto.mock_s3 -def setup_contract(): - ct = Contract( env=ENV, - parent = , - child = , - branch = , - state = , - ) - return ct - - - - ex = ExtractTransform(ENV, t_target, - -''' @moto.mock_s3 def s3_setup(): client = boto3.client('s3') - time = time.time() - file_name = 'sandwiches12345.txt' - BINARY_DATA = b'Gobias some coffee!' - CONTRACT_PATH = f'master/bluth/cornballer/raw/{file_name}' + n_time = time.time() + time_delta = 10000000 + t_file = tempfile.NamedTemporaryFile() + t_file.write(b'Gobias some coffee!') + file_name = os.path.split(t_file.name)[1] + output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev', file_name = file_name) client.create_bucket(Bucket= DEV_BUCKET) - client.put_object(Body=BINARY_DATA, Bucket=BUCKET, Key= CONTRACT_PATH, ExtraArgs={"Metadata":{"source_modified_time": str(time - 10000000)}}) - return (file_name, time,) + client.upload_file(Bucket=DEV_BUCKET, Filename= t_file.name, Key= output_contract.get_key(), ExtraArgs={"Metadata":{"source_modified_time": str(n_time - time_delta)}}) + return (t_file, output_contract, time, time_delta) -def test_push_to_s3_new_file(): +def test_push_to_s3_updated_file(): params = s3_setup() + extract = ExtractTransform() + extract.push_to_s3(tmp_dir=os.path.dirname(params[0].name), + output_contract=params[1]) + client = boto3.client('s3') + s3_time = float(client.head_object(Bucket=DEV_BUCKET, Key=Contract.get_key())['Metadata']['source_modified_time']) + assert os.stat(t_file.name).st_mtime == s3_time -def test_push_to_s3_updated_file(): +def test_push_to_s3_new_file(): pass def test_push_to_s3_not_if_older(): From a84723cabc2513721233b5ee8e0ddc8cb225d9c1 Mon Sep 17 00:00:00 2001 From: "Rayne N. Schaffer" Date: Wed, 30 Jan 2019 07:46:52 -0500 Subject: [PATCH 9/9] added extract unit tests and cleaned uo old file --- core/cli.py | 12 --- core/transforms/shared/raw/extract.py | 15 ++-- tests/unit/test_extract.py | 117 +++++++++++++++++++++++--- transforms/shared/raw/extract.py | 82 ------------------ 4 files changed, 110 insertions(+), 116 deletions(-) delete mode 100644 transforms/shared/raw/extract.py diff --git a/core/cli.py b/core/cli.py index 05ceaa0d..8b13fcf3 100644 --- a/core/cli.py +++ b/core/cli.py @@ -5,8 +5,6 @@ -from transforms.shared.raw import extract - @click.group() def cli(): # pragma: no cover pass @@ -18,16 +16,6 @@ def add(a, b): click.echo(print(a + b)) return a + b -@cli.command() -@click.argument('env',type=click.Choice(['dev'])) -@click.argument('manufacturer', type=str) -@click.argument('brand', type=str) -@click.argument('id', type=int) -def run_extract(env,manufacturer,brand,id): - repo = Repo('.') - branch_name = repo.active_branch.name - extract.test_run_extract_transform(env=env,transform_id=id, branch=branch_name, manufacturer=manufacturer, brand=brand) - @cli.command() @click.argument('env', type=click.Choice(['local'])) diff --git a/core/transforms/shared/raw/extract.py b/core/transforms/shared/raw/extract.py index 79b3dd07..7dded40a 100644 --- a/core/transforms/shared/raw/extract.py +++ b/core/transforms/shared/raw/extract.py @@ -8,8 +8,8 @@ class ExtractTransform(): def __init__(self, **kwargs) -> None: - """ performs the extraction to a given output contract. - valid kwargs: + """ Performs the extraction to a given output contract. + Valid kwargs: - env one of "dev", "prod", "uat" - transform a configuration contract instance - output_contract a contract instance @@ -37,8 +37,6 @@ def set_transform(self, transform: configuration.Transformation) -> None: def set_output_contract(self, output_contract: contract) -> None: self.output_contract = output_contract - - def run(self): for config in self.transform.extract_configurations: @@ -46,17 +44,17 @@ def run(self): remote_path = config.filesystem_path prefix = config.prefix secret_name = config.secret_name + secret_type_of = config.secret_type_of # Fetch secret from secret contract # TODO: Currently configs made for FTP only, FTP type passed in directly - source_secret = secret.Secret(name=config.secret_name,env=self.env,type_of=config.secret_type_of,mode="write") + source_secret = secret.Secret(name=secret_name,env=self.env,type_of=secret_type_of,mode="write") # Get files from remote and start pushing to s3 with tempfile.TemporaryDirectory() as tmp_dir: file_mover.get_files(tmp_dir=tmp_dir,prefix=prefix,remote_path=remote_path,secret=source_secret) self.push_to_s3(tmp_dir, self.output_contract) - def push_to_s3(self, tmp_dir: str, output_contract: contract)-> None: """ For a local file dir, push the file to s3 if it is newer or does not exist.""" self._validate_required_params() @@ -73,8 +71,9 @@ def push_to_s3(self, tmp_dir: str, output_contract: contract)-> None: output_contract.publish_raw_file(local_file_path) def _file_needs_update(self,output_contract: contract,local_file_path: str,local_file_modified_time: str)-> None: - # Check if file needs to be pushed - # File is only considered to need to be pushed if it does not exist or has been modified since last push + """ Check if file needs to be pushed + File is only considered to need to be pushed if it does not exist or has been modified since last push + """ try: s3_last_modified = output_contract.get_raw_file_metadata(local_file_path)['Metadata']['source_modified_time'] if (float(s3_last_modified) < float(local_file_modified_time)): diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index 6dd9c0be..cd3170c9 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -15,34 +15,123 @@ def s3_setup(): client = boto3.client('s3') n_time = time.time() time_delta = 10000000 - t_file = tempfile.NamedTemporaryFile() - t_file.write(b'Gobias some coffee!') - file_name = os.path.split(t_file.name)[1] - output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev', file_name = file_name) + output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev') client.create_bucket(Bucket= DEV_BUCKET) - client.upload_file(Bucket=DEV_BUCKET, Filename= t_file.name, Key= output_contract.get_key(), ExtraArgs={"Metadata":{"source_modified_time": str(n_time - time_delta)}}) - return (t_file, output_contract, time, time_delta) + t_file_old = tempfile.NamedTemporaryFile() + t_file_old.write(b'Gobias some old coffee!') + file_name_old = os.path.split(t_file_old.name)[1] + output_contract.set_file_name(file_name_old) + client.upload_file(Bucket=DEV_BUCKET, Filename= t_file_old.name, Key= output_contract.get_key(), ExtraArgs={"Metadata":{"source_modified_time": str(n_time - time_delta)}}) + + t_file_new = tempfile.NamedTemporaryFile() + t_file_new.write(b'Gobias some new coffee!') + file_name_new = os.path.split(t_file_new.name)[1] + output_contract.set_file_name(file_name_new) + client.upload_file(Bucket=DEV_BUCKET, Filename= t_file_new.name, Key= output_contract.get_key(), ExtraArgs={"Metadata":{"source_modified_time": str(n_time + time_delta)}}) + + return (t_file_old, t_file_new, output_contract, time_delta) +@moto.mock_s3 def test_push_to_s3_updated_file(): params = s3_setup() extract = ExtractTransform() - extract.push_to_s3(tmp_dir=os.path.dirname(params[0].name), - output_contract=params[1]) client = boto3.client('s3') - s3_time = float(client.head_object(Bucket=DEV_BUCKET, Key=Contract.get_key())['Metadata']['source_modified_time']) + extract.push_to_s3(tmp_dir=os.path.dirname(params[0].name), + output_contract=params[2]) + params[2].set_file_name(os.path.split(params[0].name)[1]) + s3_time = float(client.head_object(Bucket=DEV_BUCKET, Key=params[2].get_key())['Metadata']['source_modified_time']) - assert os.stat(t_file.name).st_mtime == s3_time + assert os.stat(params[0].name).st_mtime == s3_time +@moto.mock_s3 def test_push_to_s3_new_file(): - pass + params = s3_setup() + extract = ExtractTransform() + client = boto3.client('s3') + t_file = tempfile.NamedTemporaryFile() + t_file.write(b'Gobias some fresh coffee!') + extract.push_to_s3(tmp_dir=os.path.dirname(t_file.name), + output_contract=params[2]) + params[2].set_file_name(os.path.split(t_file.name)[1]) + s3_time = float(client.head_object(Bucket=DEV_BUCKET, Key=params[2].get_key())['Metadata']['source_modified_time']) + + assert os.stat(t_file.name).st_mtime == s3_time +@moto.mock_s3 def test_push_to_s3_not_if_older(): - pass + params = s3_setup() + extract = ExtractTransform() + client = boto3.client('s3') + params[2].set_file_name(os.path.split(params[1].name)[1]) + s3_time_before = float(client.head_object(Bucket=DEV_BUCKET, Key=params[2].get_key())['Metadata']['source_modified_time']) + extract.push_to_s3(tmp_dir=os.path.dirname(params[1].name), + output_contract=params[2]) + params[2].set_file_name(os.path.split(params[1].name)[1]) + s3_time_after = float(client.head_object(Bucket=DEV_BUCKET, Key=params[2].get_key())['Metadata']['source_modified_time']) + + assert s3_time_after == s3_time_before +@moto.mock_s3 def test_file_needs_update_needs_update(): - pass + params = s3_setup() + extract = ExtractTransform() + check_update = extract._file_needs_update(params[2], params[0].name, os.stat(params[0].name).st_mtime) + assert check_update == True +@moto.mock_s3 def test_file_needs_update_doesnt_need_update(): - pass + params = s3_setup() + extract = ExtractTransform() + check_update = extract._file_needs_update(params[2], params[1].name, 0) + assert check_update == False + +@moto.mock_s3 +def test_file_needs_update_doesnt_exist(): + params = s3_setup() + extract = ExtractTransform() + t_file = tempfile.NamedTemporaryFile() + t_file.write(b'Gobias some coffee!') + check_update = extract._file_needs_update(params[2], t_file, os.stat(t_file.name).st_mtime) + assert check_update == True + +def test_extract_validate_params_bad(): + with pytest.raises(ValueError) as e: + output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev') + extract = ExtractTransform(env="not-an-env",output_contract=output_contract) + extract._validate_required_params() + + assert e.type == ValueError + +def test_extract_validate_params_good(): + output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev') + transformation = C.Transformation() + extract = ExtractTransform(env="dev",output_contract=output_contract,transform=transformation) + extract._validate_required_params() + +def test_set_env_good(): + extract = ExtractTransform() + extract.set_env("dev") + + assert extract.env == "dev" + +def test_set_env_bad(): + with pytest.raises(ValueError) as e: + extract = ExtractTransform() + extract.set_env("not-an-env") + + assert e.type == ValueError + +def test_set_output_contract(): + output_contract = contract.Contract(branch='master', parent='bluth', child='cornballer',state='raw',env='dev') + extract = ExtractTransform() + extract.set_output_contract(output_contract) + + assert output_contract == extract.output_contract + +def test_set_transform(): + transformation = C.Transformation() + extract = ExtractTransform() + extract.set_transform(transformation) + assert transformation == extract.transform \ No newline at end of file diff --git a/transforms/shared/raw/extract.py b/transforms/shared/raw/extract.py deleted file mode 100644 index 53e162b2..00000000 --- a/transforms/shared/raw/extract.py +++ /dev/null @@ -1,82 +0,0 @@ -from core.helpers import configuration_mocker, file_mover -from core.models import configuration -from core import secret, contract - -import os -import tempfile - -def test_run_extract_transform(env:str,transform_id:int,branch:str,manufacturer:str,brand:str): - # TODO: This is currently using the config mocker to generate configs to run against - # When configs are ready, switch out to appropriate functions - - # Mocking configs for use in extract testing - config_mock = configuration_mocker.ConfigurationMocker() - config_mock.generate_mocks() - - session = config_mock.get_session() - - ec = configuration.ExtractConfiguration - t = configuration.Transformation - - # Start querying the extract configs - q = session.query(t).filter(t.id == transform_id) - for f_transform in q: - transform_state = str.lower(f_transform.pipeline_state.pipeline_state_type.name) - for extract in f_transform.extract_configurations: - output_contract = contract.Contract(env=env, - state=transform_state, - branch=branch, - parent=manufacturer, - child=brand - ) - extract_transform = ExtractTransform(env=env, mock_config=extract,output_contract=output_contract) - extract_transform.test_run() - - -class ExtractTransform(): - def __init__(self, env:str, mock_config, output_contract): - self.env = env - self.config=mock_config - self.output_contract = output_contract - - def test_run(self): - # Set values from extract config - remote_path = self.config.filesystem_path - prefix = self.config.prefix - secret_name = self.config.secret_name - - # Fetch secret from secret contract - # TODO: Currently configs made for FTP only, FTP type passed in directly - source_secret = secret.Secret(name=self.config.secret_name,env=self.env,type_of=self.config.secret_type_of,mode="write") - - # Get files from remote and start pushing to s3 - with tempfile.TemporaryDirectory() as tmp_dir: - file_mover.get_files(tmp_dir=tmp_dir,prefix=prefix,remote_path=remote_path,secret=source_secret) - self.push_to_s3(tmp_dir) - - - def push_to_s3(self, tmp_dir): - # Get the transformation's output contract - oc = self.output_contract - - # For each local file, see (by the set metadata) if it needs to be pushed to S3 by the constraints - for local_file in os.listdir(f"{tmp_dir}"): - local_file_path = os.path.join(tmp_dir,local_file) - local_file_modified_time = os.stat(os.path.join(tmp_dir,local_file)).st_mtime - - if (self._file_needs_update(local_file_path,local_file_modified_time)): - self.output_contract.publish_raw_file(local_file_path) - - def _file_needs_update(self,local_file_path,local_file_modified_time): - # Check if file needs to be pushed - # File is only considered to need to be pushed if it does not exist or has been modified since last push - output_contract = self.output_contract - - try: - s3_last_modified = output_contract.get_raw_file_metadata(local_file_path)['Metadata']['source_modified_time'] - if (float(s3_last_modified) < float(local_file_modified_time)): - return True - else: - return False - except: - return True