Pull buckets into env/configuration, expand edm connector testing (#1108

) * expand recipes test coverage * move buckets to configuration.py * downstream effects of moving buckets * expand publish testing, add legacy_upload * clean up templatedb publishing interaction * add dev flags to build, library, ingest workflows * add tabulate to build image reqs
NYCPlanning · Sep 6, 2024 · c3e5881 · c3e5881
1 parent 342ec15
commit c3e5881
Show file tree

Hide file tree

Showing 45 changed files with 576 additions and 240 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -56,6 +56,10 @@ on:
         description: "An optional note about this build"
         type: string
         required: false
+      dev_bucket:
+        description: "dev bucket to use in place of recipes/publishing. If name of bucket is 'de-dev-a' simply put 'a'"
+        type: string
+        required: false
   workflow_call:
     inputs:
       dataset_name:
@@ -107,6 +111,7 @@ jobs:
       plan_command: ${{ needs.health_check.outputs.plan_command }}
       logging_level: ${{ inputs.logging_level }}
       build_note: ${{ inputs.build_note }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   cbbr:
     needs: health_check
     if: inputs.dataset_name == 'cbbr' || inputs.dataset_name  == 'all'
@@ -119,6 +124,7 @@ jobs:
       plan_command: ${{ needs.health_check.outputs.plan_command }}
       logging_level: ${{ inputs.logging_level }}
       build_note: ${{ inputs.build_note }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   checkbook:
     needs: health_check
     if: inputs.dataset_name == 'checkbook' || inputs.dataset_name  == 'all'
@@ -127,6 +133,7 @@ jobs:
     with:
       image_tag: ${{ needs.health_check.outputs.tag }}
       build_name: ${{ needs.health_check.outputs.build_name }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   colp:
     needs: health_check
     if: inputs.dataset_name == 'colp' || inputs.dataset_name  == 'all'
@@ -137,6 +144,7 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   cpdb:
     needs: health_check
     if: inputs.dataset_name == 'cpdb' || inputs.dataset_name  == 'all'
@@ -147,6 +155,7 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   developments:
     needs: health_check
     if: inputs.dataset_name == 'developments' || inputs.dataset_name  == 'all'
@@ -157,6 +166,7 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   facilities:
     needs: health_check
     if: inputs.dataset_name == 'facilities' || inputs.dataset_name  == 'all'
@@ -167,6 +177,7 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   factfinder:
     needs: health_check
     if: inputs.dataset_name == 'factfinder'
@@ -176,6 +187,7 @@ jobs:
       image_tag: ${{ needs.health_check.outputs.tag }}
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   green_fast_track:
     needs: health_check
     if: inputs.dataset_name == 'green_fast_track' || inputs.dataset_name  == 'all'
@@ -186,6 +198,7 @@ jobs:
       recipe_file: ${{ inputs.recipe_file }}
       build_name: ${{ needs.health_check.outputs.build_name }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   knownprojects:
     needs: health_check
     if: inputs.dataset_name == 'knownprojects' || inputs.dataset_name  == 'all'
@@ -194,6 +207,7 @@ jobs:
     with:
       image_tag: ${{ needs.health_check.outputs.tag }}
       build_name: ${{ needs.health_check.outputs.build_name }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   pluto:
     needs: health_check
     if: inputs.dataset_name == 'pluto' || inputs.dataset_name  == 'all'
@@ -204,6 +218,7 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
   ztl:
     needs: health_check
     if: inputs.dataset_name == 'ztl' || inputs.dataset_name  == 'all'
@@ -214,3 +229,4 @@ jobs:
       build_name: ${{ needs.health_check.outputs.build_name }}
       recipe_file: ${{ inputs.recipe_file }}
       plan_command: ${{ needs.health_check.outputs.plan_command }}
+      dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }}
diff --git a/.github/workflows/cbbr_build.yml b/.github/workflows/cbbr_build.yml
@@ -21,6 +21,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        default: false
 
 jobs:
   build:
@@ -35,6 +38,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-cbbr
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/checkbook_build.yml b/.github/workflows/checkbook_build.yml
@@ -9,6 +9,9 @@ on:
       build_name:
         type: string
         required: true
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -19,6 +22,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-checkbook
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     defaults:
       run:
         shell: bash

diff --git a/.github/workflows/colp_build.yml b/.github/workflows/colp_build.yml
@@ -14,6 +14,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -28,6 +31,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-colp
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     services:
       postgis:
         image: postgis/postgis:15-3.3-alpine

diff --git a/.github/workflows/cpdb_build.yml b/.github/workflows/cpdb_build.yml
@@ -15,6 +15,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -29,6 +32,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-cpdb
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/data_library_single.yml b/.github/workflows/data_library_single.yml
@@ -21,6 +21,10 @@ on:
         type: boolean
         required: true
         default: false
+      dev_bucket:
+        description: "dev bucket to use in place of recipes. If name of bucket is 'de-dev-a' simply put 'a'"
+        type: string
+        required: false
 
 jobs:
   dataloading:
@@ -31,7 +35,9 @@ jobs:
       run:
         shell: bash
     env:
-      AWS_S3_BUCKET: edm-recipes
+      BUILD_NAME: ${{ github.head_ref || github.ref_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || 'edm-recipes' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/developments_build.yml b/.github/workflows/developments_build.yml
@@ -15,6 +15,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -26,9 +29,12 @@ jobs:
         working-directory: products/developments
     container:
       image: nycplanning/build-geosupport:${{ inputs.image_tag || 'latest' }}
-      env:
-        BUILD_ENGINE_DB: db-devdb
-        BUILD_NAME: ${{ inputs.build_name }}
+    env:
+      BUILD_ENGINE_DB: db-devdb
+      BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/facilities_build.yml b/.github/workflows/facilities_build.yml
@@ -15,6 +15,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -26,9 +29,12 @@ jobs:
         working-directory: products/facilities
     container:
       image: nycplanning/build-geosupport:${{ inputs.image_tag || 'latest' }}
-      env:
-        BUILD_ENGINE_DB: db-facilities
-        BUILD_NAME: ${{ inputs.build_name }}
+    env:
+      BUILD_ENGINE_DB: db-facilities
+      BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/factfinder_build.yml b/.github/workflows/factfinder_build.yml
@@ -12,6 +12,9 @@ on:
       recipe_file:
         type: string
         required: true
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -25,6 +28,9 @@ jobs:
       image: nycplanning/build-base:${{ inputs.image_tag || 'latest' }}
     env:
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/green_fast_track_build.yml b/.github/workflows/green_fast_track_build.yml
@@ -14,6 +14,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -28,6 +31,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-green-fast-track
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/ingest_single.yml b/.github/workflows/ingest_single.yml
@@ -24,6 +24,10 @@ on:
         type: boolean
         required: true
         default: false
+      dev_bucket:
+        description: "dev bucket to use in place of recipes. If name of bucket is 'de-dev-a' simply put 'a'"
+        type: string
+        required: false
 
 jobs:
   dataloading:
@@ -33,6 +37,11 @@ jobs:
     defaults:
       run:
         shell: bash
+    env:
+      BUILD_NAME: ${{ github.head_ref || github.ref_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/knownprojects_build.yml b/.github/workflows/knownprojects_build.yml
@@ -9,6 +9,9 @@ on:
       build_name:
         type: string
         required: true
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -23,6 +26,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: kpdb
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/pluto_build.yml b/.github/workflows/pluto_build.yml
@@ -18,6 +18,9 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 jobs:
   build:
@@ -32,6 +35,9 @@ jobs:
     env:
       BUILD_ENGINE_DB: db-pluto
       BUILD_NAME: ${{ inputs.build_name }}
+      RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+      PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+      DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
     steps:
       - uses: actions/checkout@v4
 

diff --git a/.github/workflows/template_build.yml b/.github/workflows/template_build.yml
@@ -24,12 +24,18 @@ on:
       plan_command:
         type: string
         default: recipe
+      dev_bucket:
+        type: string
+        required: false
 
 env:
   TOY_SECRET_GITHUB: ${{ secrets.TOY_SECRET }}
   BUILD_ENGINE_DB: db-template
   BUILD_NAME: ${{ inputs.build_name || github.head_ref || github.ref_name }}
   LOGGING_LEVEL: ${{ inputs.logging_level }}
+  RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }}
+  PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }}
+  DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }}
 
 jobs:
   build:

diff --git a/.github/workflows/template_test.yml b/.github/workflows/template_test.yml
@@ -12,6 +12,8 @@ env:
   TOY_SECRET_GITHUB: ${{ secrets.TOY_SECRET }}
   BUILD_ENGINE_DB: db-template
   BUILD_NAME: ${{ inputs.build_name || github.head_ref || github.ref_name }}
+  RECIPES_BUCKET: edm-recipes
+  PUBLISHING_BUCKET: edm-publishing
 
 jobs:
   unit_tests: