Merge branch 'main' into dependabot/gradle/packages/Manager/org.sprin…

…gframework.boot-3.3.0

packages/DataTransformation/README.md

@@ -24,3 +24,17 @@ Will run and return a non-zero exit code if there are any linting errors (for CI
 Will fix any linting errors it can (for local development)
 
 * `./batect lint-fix`
+
+### Generate DBT documentation
+
+The command helps in generating your project's documentation and catelog will be created.
+
+* `./batect generate-docs`
+
+### Start a webserver
+This command starts a webserver on port 8080 to serve your documentation locally and opens the documentation site in your default browser.
+
+* `./batect serve-docs`
+
+Note: Be sure to run docs generate before docs serve because the generate command produces a catalog metadata artifact that the serve command depends upon. 
+You will see an error message if the catalog is missing.

packages/DataTransformation/batect.yml

@@ -42,6 +42,20 @@ tasks:
       container: dbt-env
       command: run --profiles-dir /usr/app/dbt --target dev
 
+  generate-docs:
+    description: Generate DBT documentation
+    run:
+      container: dbt-env
+      command: docs generate --profiles-dir /usr/app/dbt --target dev
+
+  serve-docs:
+    description: Serve DBT documentation
+    run:
+      container: dbt-env
+      command: docs serve --port 8001 --profiles-dir /usr/app/dbt --target dev
+      ports:
+      - 8001:8001
+
 containers:
   dbt-env:
     image: ghcr.io/dbt-labs/dbt-postgres:1.5.8

packages/DataTransformation/models/marts/planLimits/council_plan_boundaries.sql

@@ -0,0 +1,43 @@
+{{ config(
+    materialized = 'table',
+    indexes=[
+      {'columns': ['council_id', 'source_id'], 'unique': True},
+    ]
+) }}
+
+WITH geom_source AS (
+
+  SELECT
+    council_id,
+    source_id,
+    boundary
+  FROM {{ ref('stg_council_plan_boundary_geom_source') }}
+),
+
+rec2_data AS (
+
+  SELECT
+    council_id,
+    source_id,
+    boundary
+  FROM {{ ref('council_plan_boundary_rec2_data') }}
+),
+
+geojson_data AS (
+
+  SELECT
+    council_id,
+    source_id,
+    boundary
+  FROM {{ ref('stg_council_plan_boundary_geojson_data') }}
+),
+
+combined_data AS (
+  SELECT * FROM geom_source
+  UNION ALL
+  SELECT * FROM rec2_data
+  UNION ALL
+  SELECT * FROM geojson_data
+)
+
+SELECT * FROM combined_data

packages/DataTransformation/models/marts/planLimits/council_plan_boundary_rec2_data.sql

@@ -0,0 +1,63 @@
+{{ config(
+    materialized = 'table'
+) }}
+
+WITH
+RECURSIVE
+input AS (
+  SELECT
+    council_id,
+    source_id,
+    hydro_ids,
+    excluded_hydro_ids
+  FROM
+    {{ ref('stg_council_plan_boundary_rec2_source') }}
+),
+
+expanded_input AS (
+  SELECT
+    council_id,
+    source_id,
+    UNNEST(hydro_ids) AS hydro_id
+  FROM
+    input
+),
+
+all_hydro_ids AS (
+  SELECT *
+  FROM
+    expanded_input
+  UNION ALL
+  SELECT
+    a.council_id,
+    a.source_id,
+    rv.hydro_id
+  FROM
+    {{ ref('rivers') }} AS rv
+  INNER JOIN all_hydro_ids AS a ON rv.next_hydro_id = a.hydro_id
+),
+
+with_watershed_geom AS (
+  SELECT
+    a.council_id,
+    a.source_id,
+    a.hydro_id,
+    w.geom
+  FROM
+    all_hydro_ids AS a
+  INNER JOIN {{ ref('watersheds') }} AS w ON a.hydro_id = w.hydro_id
+),
+
+as_grouped_boundaries AS (
+  SELECT
+    council_id,
+    source_id,
+    ST_UNION(geom) AS boundary
+  FROM
+    with_watershed_geom
+  GROUP BY council_id, source_id
+)
+
+SELECT *
+FROM
+  as_grouped_boundaries

packages/DataTransformation/models/marts/planlimits/_planlimits_models.yml

@@ -286,6 +286,7 @@ models:
         meta:
           data_type: varchar
 
+
   - name: water_allocation_and_usage_by_area
     columns:
       - name: date

packages/DataTransformation/models/marts/planlimits/flow_limits.sql

@@ -4,7 +4,7 @@
 
 WITH council_plan_boundaries AS (
 
-  SELECT * FROM {{ ref('stg_planlimits_council_plan_boundaries') }}
+  SELECT * FROM {{ ref('council_plan_boundaries') }}
 ),
 
 plans AS (

packages/DataTransformation/models/marts/planlimits/flow_measurement_sites.sql

@@ -4,7 +4,7 @@
 
 WITH council_plan_boundaries AS (
 
-  SELECT * FROM {{ ref('stg_planlimits_council_plan_boundaries') }}
+  SELECT * FROM {{ ref('council_plan_boundaries') }}
 ),
 
 plans AS (

packages/DataTransformation/models/marts/planlimits/groundwater_areas.sql

@@ -4,7 +4,7 @@
 
 WITH council_plan_boundaries AS (
 
-  SELECT * FROM {{ ref('stg_planlimits_council_plan_boundaries') }}
+  SELECT * FROM {{ ref('council_plan_boundaries') }}
 ),
 
 plans AS (

packages/DataTransformation/models/marts/planlimits/plan_regions.sql

@@ -4,7 +4,7 @@
 
 WITH council_plan_boundaries AS (
 
-  SELECT * FROM {{ ref('stg_planlimits_council_plan_boundaries') }}
+  SELECT * FROM {{ ref('council_plan_boundaries') }}
 ),
 
 plans AS (

packages/DataTransformation/models/marts/planlimits/surface_water_limits.sql

@@ -4,25 +4,25 @@
 
 WITH council_plan_boundaries AS (
 
-  SELECT * FROM {{ ref('stg_planlimits_council_plan_boundaries') }}
+  SELECT * FROM {{ ref('council_plan_boundaries') }}
 ),
 
-plans AS (
+     plans AS (
 
   SELECT * FROM {{ ref('plans') }}
-),
+     ),
 
-plan_regions AS (
+     plan_regions AS (
 
   SELECT * FROM {{ ref('plan_regions') }}
-),
+     ),
 
-temp_plan_regions AS (
+     temp_plan_regions AS (
 
   SELECT * FROM {{ ref('int_temp_plan_regions') }}
-),
+     ),
 
-expanded_surface_water_limits AS (
+     expanded_surface_water_limits AS (
 
   SELECT
 
@@ -34,12 +34,12 @@ expanded_surface_water_limits AS (
   FROM
     temp_plan_regions AS tpr
 
-  INNER JOIN
-    plan_regions AS pr
-    ON tpr.plan_id = pr.plan_id AND tpr.region ->> 'id' = pr.source_id
-),
+      INNER JOIN
+      plan_regions AS pr
+      ON tpr.plan_id = pr.plan_id AND tpr.region ->> 'id' = pr.source_id
+     ),
 
-id_surface_water_limits AS (
+     id_surface_water_limits AS (
 
   SELECT
 
@@ -49,9 +49,9 @@ id_surface_water_limits AS (
     ROW_NUMBER() OVER () AS id
   FROM
     expanded_surface_water_limits
-),
+     ),
 
-expanded_surface_water_sub_limits AS (
+     expanded_surface_water_sub_limits AS (
   SELECT
 
     swl.council_id,
@@ -61,9 +61,9 @@ expanded_surface_water_sub_limits AS (
 
   FROM
     id_surface_water_limits AS swl
-),
+     ),
 
-id_surface_water_sub_limits AS (
+     id_surface_water_sub_limits AS (
 
   SELECT
 
@@ -75,9 +75,9 @@ id_surface_water_sub_limits AS (
 
   FROM
     expanded_surface_water_sub_limits
-),
+     ),
 
-combined_surface_water_limits AS (
+     combined_surface_water_limits AS (
   SELECT
     id,
     plan_region_id,
@@ -90,8 +90,8 @@ combined_surface_water_limits AS (
       FROM council_plan_boundaries
 
       WHERE
-        council_id = id_surface_water_limits.council_id
-        AND surface_water_limit ->> 'boundaryId' = council_plan_boundaries.source_id
+          council_id = id_surface_water_limits.council_id
+      AND surface_water_limit ->> 'boundaryId' = council_plan_boundaries.source_id
     ) AS boundary
 
   FROM
@@ -112,12 +112,12 @@ combined_surface_water_limits AS (
       FROM council_plan_boundaries
 
       WHERE
-        council_id = id_surface_water_sub_limits.council_id
-        AND surface_water_limit ->> 'boundaryId' = council_plan_boundaries.source_id
+          council_id = id_surface_water_sub_limits.council_id
+      AND surface_water_limit ->> 'boundaryId' = council_plan_boundaries.source_id
     ) AS boundary
 
   FROM
     id_surface_water_sub_limits
-)
+     )
 
 SELECT * FROM combined_surface_water_limits

packages/DataTransformation/models/marts/planlimits/water_allocation_and_usage_by_area.sql

@@ -112,7 +112,7 @@ meter_use_by_consent_area AS (
 
   FROM
     expanded_meters_per_consent_area
-  LEFT JOIN observed_water_use_aggregated_daily AS use 
+  LEFT JOIN observed_water_use_aggregated_daily AS use
     ON date = day_observed_at AND meter = site_name
 
   WHERE meter NOT LIKE 'WAR%_Combined'

packages/DataTransformation/models/marts/planlimits/water_allocations_by_area.sql

@@ -16,7 +16,10 @@ water_allocations_by_area AS (
 
   FROM water_allocations
   WHERE
-    effective_to IS NULL
+    effective_to IS NULL 
+  AND 
+    status = 'active'
+
   GROUP BY 1
 
 ),
@@ -43,6 +46,10 @@ water_allocations_ruamahangasw AS (
     'WaiohineSW',
     'WaipouaSW'
   )
+  AND
+    effective_to IS NULL
+  AND 
+    status = 'active'
 
 )