diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e900f61f9..cb2131a76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,7 @@ on: options: - template - cbbr + - cdbg - checkbook - colp - cpdb @@ -128,6 +129,17 @@ jobs: logging_level: ${{ inputs.logging_level }} build_note: ${{ inputs.build_note }} dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }} + cdbg: + needs: health_check + if: inputs.dataset_name == 'cdbg' || inputs.dataset_name == 'all' + uses: ./.github/workflows/cdbg_build.yml + secrets: inherit + with: + image_tag: ${{ needs.health_check.outputs.tag }} + recipe_file: ${{ inputs.recipe_file }} + build_name: ${{ needs.health_check.outputs.build_name }} + plan_command: ${{ needs.health_check.outputs.plan_command }} + dev_bucket: ${{ inputs.dev_bucket && format('de-dev-{0}', inputs.dev_bucket) || '' }} checkbook: needs: health_check if: inputs.dataset_name == 'checkbook' || inputs.dataset_name == 'all' diff --git a/.github/workflows/cdbg_build.yml b/.github/workflows/cdbg_build.yml new file mode 100644 index 000000000..127b61aaa --- /dev/null +++ b/.github/workflows/cdbg_build.yml @@ -0,0 +1,76 @@ +name: CDBG - 🏗️ Build +on: + workflow_call: + inputs: + image_tag: + type: string + required: false + build_name: + type: string + required: true + recipe_file: + type: string + required: true + plan_command: + type: string + default: recipe + dev_bucket: + type: string + required: false + +jobs: + build: + name: Build CDBG + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + working-directory: products/cdbg + container: + image: nycplanning/build-base:${{ inputs.image_tag || 'latest' }} + env: + BUILD_ENGINE_DB: db-cdbg + BUILD_NAME: ${{ inputs.build_name }} + RECIPES_BUCKET: ${{ inputs.dev_bucket || 'edm-recipes' }} + PUBLISHING_BUCKET: ${{ inputs.dev_bucket || 'edm-publishing' }} + DEV_FLAG: ${{ inputs.dev_bucket && 'true' || 'false' }} + steps: + - uses: actions/checkout@v4 + + - name: Load Secrets + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + AWS_S3_ENDPOINT: "op://Data Engineering/DO_keys/AWS_S3_ENDPOINT" + AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY" + AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID" + BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url" + BUILD_ENGINE_HOST: "op://Data Engineering/EDM_DATA/server" + BUILD_ENGINE_USER: "op://Data Engineering/EDM_DATA/username" + BUILD_ENGINE_PASSWORD: "op://Data Engineering/EDM_DATA/password" + BUILD_ENGINE_PORT: "op://Data Engineering/EDM_DATA/port" + + - name: Setup build environment + working-directory: ./ + run: | + ./bash/docker_container_setup.sh + ./bash/build_env_setup.sh + + - name: Plan build + run: python3 -m dcpy.lifecycle.builds.plan ${{ inputs.plan_command }} + + - name: Dataloading + run: python -m dcpy.lifecycle.builds.load load --recipe-path ${{ inputs.recipe_file }}.lock.yml + + - name: Build + run: | + dbt debug + dbt build + + - name: Export + run: ./bash/export.sh + + - name: Upload + run: python3 -m dcpy.connectors.edm.publishing upload --product db-cdbg --acl public-read diff --git a/.github/workflows/test_helper.yml b/.github/workflows/test_helper.yml index ed08779e4..ea08e99f4 100644 --- a/.github/workflows/test_helper.yml +++ b/.github/workflows/test_helper.yml @@ -88,6 +88,7 @@ jobs: project: - green_fast_track - zoningtaxlots + - cdbg steps: - uses: actions/checkout@v4 - name: setup diff --git a/dcpy/lifecycle/ingest/templates/hud_lowmodincomebyblockgroup.yml b/dcpy/lifecycle/ingest/templates/hud_lowmodincomebyblockgroup.yml new file mode 100644 index 000000000..1ff10dd6a --- /dev/null +++ b/dcpy/lifecycle/ingest/templates/hud_lowmodincomebyblockgroup.yml @@ -0,0 +1,60 @@ +id: hud_lowmodincomebyblockgroup +acl: public-read + +attributes: + name: HUD Low to Moderate Income Population by Block Group + description: >- + This particular version of this dataset has come from OMB. We haven't fully assessed whether + we could just pull from the linked feature service instead and filter to NYC or if OMB has + done any other preprocessing. + + This service identifies U.S. Census Block Groups in which 51% or more of the households earn + less than 80 percent of the Area Median Income (AMI). The Community Development Block Grant + (CDBG) program requires that each CDBG funded activity must either principally benefit low- + and moderate-income persons, aid in the prevention or elimination of slums or blight, or meet + a community development need having a particular urgency because existing conditions pose a + serious and immediate threat to the health or welfare of the community and other financial + resources are not available to meet that need. With respect to activities that principally + benefit low- and moderate-income persons, at least 51 percent of the activity's beneficiaries + must be low and moderate income. + + The Community Development Block Grant (CDBG) program requires that each CDBG funded activity + must either principally benefit low- and moderate-income persons, aid in the prevention or + elimination of slums or blight, or meet a community development need having a particular urgency + because existing conditions pose a serious and immediate threat to the health or welfare of + the community and other financial resources are not available to meet that need. With respect + to activities that principally benefit low- and moderate-income persons, at least 51 percent + of the activity's beneficiaries must be low and moderate income. For CDBG, a person is considered + to be of low income only if he or she is a member of a household whose income would qualify as + "very low income" under the Section 8 Housing Assistance Payments program. Generally, these + Section 8 limits are based on 50% of area median. Similarly, CDBG moderate income relies on + Section 8 "lower income" limits, which are generally tied to 80% of area median. These data + are from the 2011-2015 American Community Survey (ACS). + url: https://hudgis-hud.opendata.arcgis.com/datasets/HUD::low-to-moderate-income-population-by-block-group/about + +ingestion: + source: + type: s3 + bucket: edm-recipes + key: inbox/omb/20241227/ACS-2020-Low-Mod-Summarized-All-Block-Groups-2023.csv + file_format: + type: csv + +#columns: +#- id: CDBGUOGID +#- id: GEOID +#- id: CDBGNAME +#- id: STUSAB +#- id: STATE +#- id: CDBGTYPE +#- id: Geoname +#- id: COUNTY +#- id: TRACT +#- id: BLKGRP +#- id: LOW +#- id: LOWMOD +#- id: LMMI +#- id: LOWMODUNIV +#- id: LOWMOD_PCT +#- id: MOE_LOWMODPCT +#- id: Column1 diff --git a/products/cdbg/bash/export.sh b/products/cdbg/bash/export.sh new file mode 100755 index 000000000..1650a4476 --- /dev/null +++ b/products/cdbg/bash/export.sh @@ -0,0 +1,22 @@ +#!/bin/bash +source ../../bash/utils.sh +set_error_traps + +rm -rf output + +echo "Export product tables" +mkdir -p output && ( + cd output + + echo "Copy metadata files" + cp ../source_data_versions.csv . + cp ../build_metadata.json . + + echo "export cdbg_block_groups.csv ..." + csv_export cdbg_block_groups cdbg_block_groups + + echo "export cdbg_tracts.csv ..." + csv_export cdbg_tracts cdbg_tracts +) + +zip -r output/output.zip output diff --git a/products/cdbg/dbt_project.yml b/products/cdbg/dbt_project.yml new file mode 100644 index 000000000..35b7e354b --- /dev/null +++ b/products/cdbg/dbt_project.yml @@ -0,0 +1,21 @@ +name: "cdbg" + +profile: "dcp-de-postgres" + +model-paths: ["models"] + +tests: + +store_failures: true + schema: "_tests" + +models: + cdbg: + staging: + +materialized: view + intermediate: + +materialized: table + product: + +materialized: table + +flags: + fail-fast: true diff --git a/products/cdbg/macros/test_sum.sql b/products/cdbg/macros/test_sum.sql new file mode 100644 index 000000000..ffc9f76df --- /dev/null +++ b/products/cdbg/macros/test_sum.sql @@ -0,0 +1,9 @@ +{% test sum_by(model, group_by, target_column, val, precision=4) %} + +SELECT + {{ group_by }}, sum({{ target_column }}) AS sum, array_agg( {{target_column }}) AS vals +FROM {{ model }} +GROUP BY {{ group_by }} +HAVING round(sum({{ target_column }})::numeric, {{ precision }}) <> {{ val }} + +{% endtest %} diff --git a/products/cdbg/models/_sources.yml b/products/cdbg/models/_sources.yml new file mode 100644 index 000000000..47127204f --- /dev/null +++ b/products/cdbg/models/_sources.yml @@ -0,0 +1,17 @@ +version: 2 + +sources: + - name: recipe_sources + schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}" + tables: + - name: dcp_mappluto_clipped + columns: + - name: bbl + tests: + - not_null + - name: wkb_geometry + tests: + - not_null + - name: dcp_cb2020_wi + - name: dcp_ct2020_wi + - name: hud_lowmodincomebyblockgroup diff --git a/products/cdbg/models/intermediate/_intermediate_models.yml b/products/cdbg/models/intermediate/_intermediate_models.yml new file mode 100644 index 000000000..d5937f1f9 --- /dev/null +++ b/products/cdbg/models/intermediate/_intermediate_models.yml @@ -0,0 +1,68 @@ +version: 2 + +models: +- name: int__block_groups + description: residential area and low-to-moderate income data aggregated by census block group + columns: + - name: geoid + tests: [unique, not_null] + - name: borough_name + - name: tract + - name: block_group + - name: total_floor_area + - name: residential_floor_area + - name: residential_floor_area_percentage + - name: total_population + - name: lowmod_population + - name: lowmod_population_percentage + +- name: int__lot_block_groups_details + description: int__lot_block_groups joined to pluto for lot info + columns: + - name: bbl + - name: block_group_geoid + - name: overlap_ratio + - name: bldgarea + - name: bldgarea_in_block_group + - name: resarea + - name: resarea_in_block_group + +- name: int__lot_block_groups_raw + description: unique intersections of pluto lots and census block groups with proportion of lot in block group + columns: + - name: bbl + - name: block_group_geoid + - name: overlap_ratio + +- name: int__lot_block_groups + description: >- + unique intersections of pluto lots and census block groups with proportion of lot in block group, + corrected to assign lots fully to block groups that contain 90%+ of a lot + columns: + - name: bbl + - name: block_group_geoid + - name: overlap_ratio + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [bbl, block_group_geoid] + - sum_by: + group_by: bbl + target_column: overlap_ratio + val: 1 + config: + severity: warn + +- name: int__tracts + description: residential area and low-to-moderate income data aggregated by census tract + columns: + - name: geoid + tests: [unique, not_null] + - name: borough_name + - name: tract + - name: block_group + - name: total_floor_area + - name: residential_floor_area + - name: residential_floor_area_percentage + - name: total_population + - name: lowmod_population + - name: lowmod_population_percentage diff --git a/products/cdbg/models/intermediate/int__block_groups.sql b/products/cdbg/models/intermediate/int__block_groups.sql new file mode 100644 index 000000000..955abd722 --- /dev/null +++ b/products/cdbg/models/intermediate/int__block_groups.sql @@ -0,0 +1,39 @@ +WITH lot_block_groups AS ( + SELECT * FROM {{ ref("int__lot_block_groups_details") }} +), + +block_groups_income AS ( + SELECT * FROM {{ ref("stg__low_mod_by_block_group") }} +), + +block_groups_floor_area AS ( + SELECT + block_group_geoid AS geoid, + sum(bldgarea_in_block_group) AS total_floor_area, + sum(resarea_in_block_group) AS residential_floor_area + FROM lot_block_groups + GROUP BY geoid +), + +block_group_details AS ( + SELECT + block_groups_floor_area.geoid, + block_groups_income.boro AS borough_name, + block_groups_income.tract, + block_groups_income.block_group, + total_floor_area, + residential_floor_area, + CASE + WHEN total_floor_area = 0 + THEN 0 + ELSE (residential_floor_area / total_floor_area) * 100 + END AS residential_floor_area_percentage, + block_groups_income.total_population, + block_groups_income.lowmod_population AS low_mod_income_population, + block_groups_income.lowmod_pct AS low_mod_income_population_percentage + FROM block_groups_floor_area + LEFT JOIN block_groups_income + ON block_groups_floor_area.geoid = block_groups_income.geoid +) + +SELECT * FROM block_group_details diff --git a/products/cdbg/models/intermediate/int__lot_block_groups.sql b/products/cdbg/models/intermediate/int__lot_block_groups.sql new file mode 100644 index 000000000..95a710b75 --- /dev/null +++ b/products/cdbg/models/intermediate/int__lot_block_groups.sql @@ -0,0 +1,35 @@ +WITH lot_block_groups AS ( + SELECT + bbl, + block_group_geoid, + overlap_ratio + FROM {{ ref("int__lot_block_groups_raw") }} +), + +valid_lot_block_groups AS ( + SELECT * FROM lot_block_groups + WHERE overlap_ratio IS NOT null +), + +lots_easy AS ( + SELECT + bbl, + block_group_geoid, + 1 AS overlap_ratio + FROM valid_lot_block_groups + WHERE overlap_ratio > 0.9 +), + +lots_split AS ( + SELECT * + FROM valid_lot_block_groups + WHERE bbl NOT IN (SELECT bbl FROM lots_easy) +), + +lots AS ( + SELECT * FROM lots_easy + UNION ALL + SELECT * FROM lots_split +) + +SELECT * FROM lots diff --git a/products/cdbg/models/intermediate/int__lot_block_groups_details.sql b/products/cdbg/models/intermediate/int__lot_block_groups_details.sql new file mode 100644 index 000000000..9150e36cc --- /dev/null +++ b/products/cdbg/models/intermediate/int__lot_block_groups_details.sql @@ -0,0 +1,37 @@ +WITH lot_block_groups AS ( + SELECT * FROM {{ ref("int__lot_block_groups") }} +), + +pluto AS ( + SELECT + bbl, + bldgarea, + resarea + FROM {{ source("recipe_sources", "dcp_mappluto_clipped") }} +), + +details AS ( + SELECT + pluto.bbl, + lot_block_groups.block_group_geoid, + pluto.bldgarea, + pluto.resarea, + lot_block_groups.overlap_ratio + FROM lot_block_groups + LEFT JOIN pluto + ON lot_block_groups.bbl = pluto.bbl +), + +ratio_details AS ( + SELECT + bbl, + block_group_geoid, + overlap_ratio, + bldgarea, + bldgarea * overlap_ratio AS bldgarea_in_block_group, + resarea, + resarea * overlap_ratio AS resarea_in_block_group + FROM details +) + +SELECT * FROM ratio_details diff --git a/products/cdbg/models/intermediate/int__lot_block_groups_raw.sql b/products/cdbg/models/intermediate/int__lot_block_groups_raw.sql new file mode 100644 index 000000000..5e0284e37 --- /dev/null +++ b/products/cdbg/models/intermediate/int__lot_block_groups_raw.sql @@ -0,0 +1,51 @@ +WITH pluto AS ( + SELECT + bbl, + wkb_geometry + FROM {{ source("recipe_sources", "dcp_mappluto_clipped") }} +), + +block_groups AS ( + SELECT + geoid, + geom + FROM {{ ref("stg__census_block_groups") }} +), + +lot_block_group_intersections AS ( + SELECT + pluto.bbl, + pluto.wkb_geometry AS lot_geometry, + ST_AREA(pluto.wkb_geometry) AS lot_area_sqft, + block_groups.geoid AS block_group_geoid, + block_groups.geom AS block_group_geometry + FROM pluto + LEFT JOIN block_groups + ON ST_INTERSECTS(pluto.wkb_geometry, block_groups.geom) +), + +intersection_calculations AS ( + SELECT + bbl, + lot_geometry, + lot_area_sqft, + block_group_geoid, + block_group_geometry, + ST_AREA( + CASE + WHEN ST_COVEREDBY(lot_geometry, block_group_geometry) THEN lot_geometry + ELSE ST_INTERSECTION(lot_geometry, block_group_geometry) + END + ) AS area_of_intersection_sqft + FROM lot_block_group_intersections +), + +intersection_ratios AS ( + SELECT + bbl, + block_group_geoid, + area_of_intersection_sqft / lot_area_sqft AS overlap_ratio + FROM intersection_calculations +) + +SELECT * FROM intersection_ratios diff --git a/products/cdbg/models/intermediate/int__tracts.sql b/products/cdbg/models/intermediate/int__tracts.sql new file mode 100644 index 000000000..760783824 --- /dev/null +++ b/products/cdbg/models/intermediate/int__tracts.sql @@ -0,0 +1,36 @@ +WITH block_groups AS ( + SELECT + *, + left(geoid, -1) AS tract_id + FROM {{ ref("int__block_groups") }} +), + +tracts AS ( + SELECT + tract_id AS geoid, + max(borough_name) AS borough_name, + sum(total_floor_area) AS total_floor_area, + sum(residential_floor_area) AS residential_floor_area, + sum(total_population) AS total_population, + sum(low_mod_income_population) AS low_mod_income_population + FROM block_groups + GROUP BY tract_id +), + +tracts_calculation AS ( + SELECT + *, + CASE + WHEN total_floor_area = 0 + THEN 0 + ELSE (residential_floor_area / total_floor_area) * 100 + END AS residential_floor_area_percentage, + CASE + WHEN total_population = 0 + THEN 0 + ELSE (low_mod_income_population / total_population) * 100 + END AS low_mod_income_population_percentage + FROM tracts +) + +SELECT * FROM tracts_calculation diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml new file mode 100644 index 000000000..0ffacd967 --- /dev/null +++ b/products/cdbg/models/product/_product_models.yml @@ -0,0 +1,155 @@ +version: 2 + +models: + - name: cdbg_tracts + description: Census tracts and their Community Development Block Grant (CDBG) eligibility details + config: + contract: + enforced: false + + columns: + - name: geoid + data_type: string + tests: + - not_null + - unique + + - name: borough_name + data_type: string + tests: [not_null] + +# - name: borough_code +# data_type: integer +# tests: [not_null] +# +# - name: borough_tract +# data_type: string +# tests: [not_null] +# +# - name: tract +# data_type: string +# tests: [not_null] + + - name: total_floor_area + data_type: integer + tests: [not_null] + + - name: residential_floor_area + data_type: integer + tests: [not_null] + + - name: residential_floor_area_percentage + data_type: float + tests: [not_null] + + - name: low_mod_income_population + data_type: integer + tests: [not_null] + + - name: low_mod_income_population_percentage + data_type: float + tests: [not_null] + + - name: eligibility_flag + data_type: boolean + tests: [not_null] + + - name: eligibility + data_type: string + tests: [not_null] + + - name: cdbg_block_groups + description: Census block groups and their Community Development Block Grant (CDBG) eligibility details + config: + contract: + enforced: false + + columns: + - name: geoid + data_type: string + tests: + - not_null + - unique + + - name: borough_name + data_type: string + tests: [not_null] + +# - name: borough_code +# data_type: integer +# tests: [not_null] +# +# - name: borough_tract +# data_type: string +# tests: [not_null] +# +# - name: tract +# data_type: string +# tests: [not_null] +# +# - name: borough_tract_block_group +# data_type: string +# tests: [not_null] +# +# - name: block_group +# data_type: string +# tests: [not_null] + + - name: total_floor_area + data_type: integer + tests: [not_null] + + - name: residential_floor_area + data_type: integer + tests: [not_null] + + - name: residential_floor_area_percentage + data_type: float + tests: [not_null] + + - name: low_mod_income_population + data_type: integer + tests: [not_null] + + - name: low_mod_income_population_percentage + data_type: float + tests: [not_null] + + - name: eligibility_flag + data_type: boolean + tests: [not_null] + + - name: eligibility + data_type: string + tests: [not_null] + +# - name: cdbg_borough +# description: Borough and city-wide Community Development Block Grant (CDBG) details +# config: +# contract: +# enforced: true +# +# columns: +# - name: borough_name +# data_type: string +# tests: [not_null] +# +# - name: total_floor_area +# data_type: integer +# tests: [not_null] +# +# - name: residential_floor_area +# data_type: integer +# tests: [not_null] +# +# - name: residential_floor_area_percentage +# data_type: float +# tests: [not_null] +# +# - name: low_mod_income_population +# data_type: integer +# tests: [not_null] +# +# - name: low_mod_income_population_percentage +# data_type: float +# tests: [not_null] diff --git a/products/cdbg/models/product/cdbg_block_groups.sql b/products/cdbg/models/product/cdbg_block_groups.sql new file mode 100644 index 000000000..1934d6034 --- /dev/null +++ b/products/cdbg/models/product/cdbg_block_groups.sql @@ -0,0 +1,22 @@ +WITH block_groups AS ( + SELECT * FROM {{ ref("int__block_groups") }} +), + +eligibility_calculation AS ( + SELECT + *, + low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag + FROM block_groups +), + +eligibility AS ( + SELECT + *, + CASE + WHEN eligibility_flag THEN 'CD Eligible' + ELSE 'Ineligible' + END AS eligibility + FROM eligibility_calculation +) + +SELECT * FROM eligibility diff --git a/products/cdbg/models/product/cdbg_tracts.sql b/products/cdbg/models/product/cdbg_tracts.sql new file mode 100644 index 000000000..374b14005 --- /dev/null +++ b/products/cdbg/models/product/cdbg_tracts.sql @@ -0,0 +1,22 @@ +WITH tracts AS ( + SELECT * FROM {{ ref("int__tracts") }} +), + +eligibility_calculation AS ( + SELECT + *, + low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag + FROM tracts +), + +eligibility AS ( + SELECT + *, + CASE + WHEN eligibility_flag THEN 'CD Eligible' + ELSE 'Ineligible' + END AS eligibility + FROM eligibility_calculation +) + +SELECT * FROM eligibility diff --git a/products/cdbg/models/staging/_staging_models.yml b/products/cdbg/models/staging/_staging_models.yml new file mode 100644 index 000000000..dfce87a32 --- /dev/null +++ b/products/cdbg/models/staging/_staging_models.yml @@ -0,0 +1,5 @@ +version: 2 + +models: + - name: stg__census_block_groups + - name: stg__low_mod_by_block_group diff --git a/products/cdbg/models/staging/stg__census_block_groups.sql b/products/cdbg/models/staging/stg__census_block_groups.sql new file mode 100644 index 000000000..a66c52538 --- /dev/null +++ b/products/cdbg/models/staging/stg__census_block_groups.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'table', + indexes=[ + {'columns': ['geom'], 'type': 'gist'}, + {'columns': ['geoid']}, + ] +) }} + +WITH census_blocks AS ( + SELECT + left(geoid, 12) AS block_group_geoid, + * + FROM {{ source("recipe_sources", "dcp_cb2020_wi") }} +) +SELECT + borocode, + boroname, + ct2020, + block_group_geoid AS geoid, + st_union(wkb_geometry) AS geom +FROM census_blocks +GROUP BY + borocode, + boroname, + ct2020, + block_group_geoid diff --git a/products/cdbg/models/staging/stg__low_mod_by_block_group.sql b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql new file mode 100644 index 000000000..190f3a8d4 --- /dev/null +++ b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql @@ -0,0 +1,14 @@ +{{ config( + materialized = 'table', + indexes=[{'columns': ['geoid']}] +) }} + +SELECT + "GEOID"::text AS geoid, -- TODO: coerce to text in ingest + "BORO" AS boro, + "TRACT" AS tract, + "BLKGRP" AS block_group, + REPLACE("LOWMODUNIV", ',', '')::numeric AS total_population, + REPLACE("LOWMOD", ',', '')::numeric AS lowmod_population, + RTRIM("LOWMOD_PCT", '%')::numeric AS lowmod_pct +FROM {{ source("recipe_sources", "hud_lowmodincomebyblockgroup") }} diff --git a/products/cdbg/packages.yml b/products/cdbg/packages.yml new file mode 100644 index 000000000..e124278c7 --- /dev/null +++ b/products/cdbg/packages.yml @@ -0,0 +1,5 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.1.1 + - package: calogica/dbt_expectations + version: 0.10.3 \ No newline at end of file diff --git a/products/cdbg/profiles.yml b/products/cdbg/profiles.yml new file mode 100644 index 000000000..671252586 --- /dev/null +++ b/products/cdbg/profiles.yml @@ -0,0 +1,11 @@ +dcp-de-postgres: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('BUILD_ENGINE_HOST') }}" + user: "{{ env_var('BUILD_ENGINE_USER') }}" + password: "{{ env_var('BUILD_ENGINE_PASSWORD') }}" + port: "{{ env_var('BUILD_ENGINE_PORT') | as_number }}" + dbname: "{{ env_var('BUILD_ENGINE_DB') }}" + schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}" diff --git a/products/cdbg/recipe.yml b/products/cdbg/recipe.yml new file mode 100644 index 000000000..b944ba3b7 --- /dev/null +++ b/products/cdbg/recipe.yml @@ -0,0 +1,9 @@ +name: Community Development Block Grant +product: db-cdbg +inputs: + missing_versions_strategy: find_latest + datasets: + - name: dcp_mappluto_clipped + - name: dcp_cb2020_wi # maybe not needed. including for now in case it's helpful for block groups + - name: dcp_ct2020_wi + - name: hud_lowmodincomebyblockgroup