Merge branch 'develop' into m/core-604/add_or_update_ds

.github/workflows/ci.yml

@@ -336,6 +336,13 @@ jobs:
       GX_CLOUD_ORGANIZATION_ID: ${{secrets.MERCURY_ORGANIZATION_ID}}
       GX_CLOUD_ACCESS_TOKEN: ${{secrets.MERCURY_ACCESS_TOKEN}}
       LD_SDK_KEY: ${{secrets.LD_SDK_KEY}}
+      GX_SCHEDULER_LAMBDA_ARN: ${{secrets.GX_SCHEDULER_LAMBDA_ARN}}
+      GX_SCHEDULER_EXECUTION_ROLE_ARN: ${{secrets.GX_SCHEDULER_EXECUTION_ROLE_ARN}}
+      GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}}
+      LOCALSTACK_AWS_ENDPOINT_URL: ${{secrets.LOCALSTACK_AWS_ENDPOINT_URL}}
+      LOCALSTACK_AWS_DEFAULT_REGION: ${{secrets.LOCALSTACK_AWS_DEFAULT_REGION}}
+      LOCALSTACK_AWS_SECRET_ACCESS_KEY: ${{secrets.LOCALSTACK_AWS_SECRET_ACCESS_KEY}}
+      LOCALSTACK_AWS_ACCESS_KEY_ID: ${{secrets.LOCALSTACK_AWS_ACCESS_KEY_ID}}
       PACT_DO_NOT_TRACK: true
       SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}}
       SNOWFLAKE_CI_USER_PASSWORD: ${{secrets.SNOWFLAKE_CI_USER_PASSWORD}}

.gitignore

@@ -157,3 +157,6 @@ tests/integration/cloud/rest_contracts/pacts
 
 # Local Netlify folder
 .netlify
+
+# mercury container volume
+assets/docker/mercury/volume/

assets/docker/mercury/docker-compose.yml

@@ -60,6 +60,14 @@ services:
       AUTH0_MERCURY_API_CLIENT_SECRET: ${AUTH0_MERCURY_API_CLIENT_SECRET}
       SENDGRID_API_KEY: fake_key
       LD_SDK_KEY: ${LD_SDK_KEY}
+      GX_SCHEDULER_LAMBDA_ARN: ${GX_SCHEDULER_LAMBDA_ARN}
+      GX_SCHEDULER_EXECUTION_ROLE_ARN: ${GX_SCHEDULER_EXECUTION_ROLE_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_REGION: ${LOCALSTACK_AWS_DEFAULT_REGION}
+      AWS_ENDPOINT_URL: ${LOCALSTACK_AWS_ENDPOINT_URL}
+      AWS_DEFAULT_REGION: ${LOCALSTACK_AWS_DEFAULT_REGION}
+      AWS_SECRET_ACCESS_KEY: ${LOCALSTACK_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${LOCALSTACK_AWS_ACCESS_KEY_ID}
     # TODO replace migtrations with ./scripts/automated_migrations.sh
     entrypoint:
       - /bin/bash
@@ -79,6 +87,8 @@ services:
     depends_on:
       db-provisioner:
         condition: service_completed_successfully
+      localstack:
+        condition: service_healthy
   mercury-service-api-v1:
     image: 258143015559.dkr.ecr.us-east-1.amazonaws.com/mercury/api:v1api-latest
     platform: linux/amd64
@@ -100,19 +110,27 @@ services:
       AUTH0_MERCURY_API_CLIENT_SECRET: ${AUTH0_MERCURY_API_CLIENT_SECRET}
       SENDGRID_API_KEY: fake_key
       LD_SDK_KEY: ${LD_SDK_KEY}
+      GX_SCHEDULER_LAMBDA_ARN: ${GX_SCHEDULER_LAMBDA_ARN}
+      GX_SCHEDULER_EXECUTION_ROLE_ARN: ${GX_SCHEDULER_EXECUTION_ROLE_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_REGION: ${LOCALSTACK_AWS_DEFAULT_REGION}
+      AWS_ENDPOINT_URL: ${LOCALSTACK_AWS_ENDPOINT_URL}
+      AWS_DEFAULT_REGION: ${LOCALSTACK_AWS_DEFAULT_REGION}
+      AWS_SECRET_ACCESS_KEY: ${LOCALSTACK_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${LOCALSTACK_AWS_ACCESS_KEY_ID}
     depends_on:
       mercury-service-api:
         condition: service_started
       db-provisioner:
         condition: service_completed_successfully
+      localstack:
+        condition: service_healthy
     entrypoint:
       - /bin/bash
       - -c
       - |
-        sed -i 's/localhost/db/g' /app/services/ge_cloud/mercury/alembic.ini
-        cd /app/services/ge_cloud/mercury
-        alembic upgrade head
         cd /app/services/ge_cloud
+        ./scripts/localstack_init.sh
         gunicorn "mercury.api_v1:get_app()" -c ./fastapi-gunicorn.conf.py
   nginx:
     image: ${ECR_PULL_THROUGH_REPOSITORY_URL}library/nginx:latest
@@ -124,3 +142,16 @@ services:
     depends_on:
       - mercury-service-api
       - mercury-service-api-v1
+  localstack:
+    container_name: localstack
+    image: localstack/localstack
+    ports:
+      - "127.0.0.1:4566:4566" # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559" # external services port range
+    environment:
+      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+      DEBUG: ${DEBUG:-0}
+      SERVICES: scheduler,lambda,sts,iam,ssm,events
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

tests/datasource/fluent/test_pandas_datasource.py

@@ -3,6 +3,7 @@
 import copy
 import inspect
 import logging
+import os
 import pathlib
 import uuid
 from pprint import pformat as pf
@@ -474,17 +475,26 @@ def test_read_dataframe(empty_data_context: AbstractDataContext, test_df_pandas:
 @pytest.mark.cloud
 def test_cloud_get_csv_asset_not_in_memory(valid_file_path: pathlib.Path):
     # this test runs end-to-end in a real Cloud Data Context
-    context = gx.get_context(mode="cloud")
-    csv_asset_name = f"DA_{uuid.uuid4().hex}"
-    datasource = context.data_sources.pandas_default
-    _ = datasource.add_csv_asset(
-        name=csv_asset_name,
-        filepath_or_buffer=valid_file_path,
+    context = gx.get_context(
+        mode="cloud",
+        cloud_base_url=os.environ.get("GX_CLOUD_BASE_URL"),
+        cloud_organization_id=os.environ.get("GX_CLOUD_ORGANIZATION_ID"),
+        cloud_access_token=os.environ.get("GX_CLOUD_ACCESS_TOKEN"),
     )
-    csv_asset = datasource.get_asset(name=csv_asset_name)
-    csv_asset.build_batch_request()
+    datasource_name = f"DS_{uuid.uuid4().hex}"
+    csv_asset_name = f"DA_{uuid.uuid4().hex}"
+    datasource = context.data_sources.add_pandas(name=datasource_name)
+    try:
+        _ = datasource.add_csv_asset(
+            name=csv_asset_name,
+            filepath_or_buffer=valid_file_path,
+        )
+        csv_asset = datasource.get_asset(name=csv_asset_name)
+        csv_asset.build_batch_request()
 
-    assert csv_asset_name not in context.data_sources.all()._in_memory_data_assets
+        assert csv_asset_name not in context.data_sources.all()._in_memory_data_assets
+    finally:
+        context.data_sources.delete(name=datasource_name)
 
 
 @pytest.mark.filesystem

tests/integration/cloud/end_to_end/conftest.py

@@ -60,7 +60,7 @@ def datasource_name(
     # in that case, we create one simply to test get and delete
     try:
         _ = context.data_sources.get(name=datasource_name)
-    except ValueError:
+    except KeyError:
         _ = context.data_sources.add_pandas(name=datasource_name)
         context.data_sources.get(name=datasource_name)
     context.delete_datasource(name=datasource_name)