Repeatable Migrations for Views, Roles, Functions

api/_sql/R__01_roles.sql

@@ -0,0 +1,109 @@
+--------
+-- ROLES
+--------
+-- For a production-ready deployment scenario, the role 'cumulus_user' with a complicated selected password
+-- should already exist, having been created when the database was stood-up.
+-- The statement below is used to create database user for developing locally with Docker Compose with a
+-- simple password ('cumulus_pass'). https://stackoverflow.com/questions/8092086/create-postgresql-role-user-if-it-doesnt-exist
+DO $$
+BEGIN
+  CREATE USER cumulus_user WITH ENCRYPTED PASSWORD 'cumulus_pass';
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role cumulus_user -- it already exists';
+END
+$$;
+
+-- Role cumulus_reader;
+DO $$
+BEGIN
+  CREATE ROLE cumulus_reader;
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role cumulus_reader -- it already exists';
+END
+$$;
+
+-- Role cumulus_writer
+DO $$
+BEGIN
+  CREATE ROLE cumulus_writer;
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role cumulus_writer -- it already exists';
+END
+$$;
+
+-- Role postgis_reader
+DO $$
+BEGIN
+  CREATE ROLE postgis_reader;
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role postgis_reader -- it already exists';
+END
+$$;
+
+-- Role postgis_reader
+GRANT SELECT ON geometry_columns TO postgis_reader;
+GRANT SELECT ON geography_columns TO postgis_reader;
+GRANT SELECT ON spatial_ref_sys TO postgis_reader;
+
+-- Grant Permissions to cumulus_user
+GRANT postgis_reader TO cumulus_user;
+GRANT cumulus_reader TO cumulus_user;
+GRANT cumulus_writer TO cumulus_user;
+
+-- Set Search Path
+ALTER ROLE cumulus_user SET search_path TO cumulus,topology,public;
+
+-- Grant Schema Usage to cumulus_user
+GRANT USAGE ON SCHEMA cumulus TO cumulus_user;
+
+-- Role cumulus_reader
+GRANT SELECT ON
+    area,
+    area_group,
+    area_group_product_statistics_enabled,
+    config,
+    profile,
+    profile_token,
+    office,
+    parameter,
+    unit,
+    product,
+    productfile,
+    product_tags,
+    suite,
+    tag,
+    acquirable,
+    acquirablefile,
+    download_status,
+    download,
+    download_product,
+    watershed,
+    my_watersheds,
+    watershed_roles
+TO cumulus_reader;
+
+-- Role cumulus_writer
+GRANT INSERT,UPDATE,DELETE ON
+    area,
+    area_group,
+    area_group_product_statistics_enabled,
+    config,
+    profile,
+    profile_token,
+    office,
+    parameter,
+    unit,
+    product,
+    productfile,
+    suite,
+    tag,
+    product_tags,
+    acquirable,
+    acquirablefile,
+    download_status,
+    download,
+    download_product,
+    watershed,
+    watershed_roles,
+    my_watersheds
+TO cumulus_writer;

api/_sql/R__02_views_profiles.sql

@@ -0,0 +1,23 @@
+-- v_profile
+-- create statement here because watershed and watershed_roles
+-- tables must exist before creating this view
+CREATE OR REPLACE VIEW v_profile AS (
+    WITH roles_by_profile AS (
+        SELECT profile_id,
+               array_agg(UPPER(b.slug || '.' || c.name)) AS roles
+        FROM watershed_roles a
+        INNER JOIN watershed b ON a.watershed_id = b.id AND NOT b.deleted
+        INNER JOIN role      c ON a.role_id    = c.id
+        GROUP BY profile_id
+    )
+    SELECT p.id,
+           p.edipi,
+           p.username,
+           p.email,
+           p.is_admin,
+           COALESCE(r.roles,'{}') AS roles
+    FROM profile p
+    LEFT JOIN roles_by_profile r ON r.profile_id = p.id
+);
+
+GRANT SELECT ON v_profile TO cumulus_reader;

api/_sql/R__03_views_watersheds.sql

@@ -0,0 +1,59 @@
+
+CREATE OR REPLACE VIEW v_watershed AS (
+    SELECT w.id,
+           w.slug,
+           w.name,
+           w.geometry AS geometry,
+           COALESCE(ag.area_groups, '{}') AS area_groups,
+           f.symbol AS office_symbol
+	FROM   watershed w
+	LEFT JOIN (
+		SELECT array_agg(id) as area_groups, watershed_id
+		FROM area_group
+		GROUP BY watershed_id
+	) ag ON ag.watershed_id = w.id
+    LEFT JOIN office f ON w.office_id = f.id
+	WHERE NOT w.deleted
+);
+
+
+-- Basins; Projected to EPSG 5070
+CREATE OR REPLACE VIEW v_area_5070 AS (
+	SELECT id,
+		slug,
+		name,
+			ST_SnapToGrid(
+				ST_Transform(
+					geometry,
+					'+proj=aea +lat_0=23 +lon_0=-96 +lat_1=29.5 +lat_2=45.5 +x_0=0 +y_0=0 +datum=NAD83 +units=us-ft +no_defs',
+					5070
+				),
+				1
+			) AS geometry
+	FROM area
+);
+
+-- v_watershed_roles
+CREATE OR REPLACE VIEW v_watershed_roles AS (
+    SELECT a.id,
+           a.profile_id,
+           b.edipi,
+           b.username,
+           b.email,
+           b.is_admin,
+           c.id AS watershed_id,
+           r.id   AS role_id,
+           r.name AS role,
+           UPPER(c.slug || '.' || r.name) AS rolename
+    FROM watershed_roles a
+    INNER JOIN profile b ON b.id = a.profile_id
+    INNER JOIN watershed c ON c.id = a.watershed_id AND NOT c.deleted
+    INNER JOIN role    r ON r.id = a.role_id
+    ORDER BY username, role
+);
+
+GRANT SELECT ON
+    v_area_5070,
+    v_watershed,
+    v_watershed_roles
+TO cumulus_reader;

api/_sql/R__04_views_products.sql

@@ -0,0 +1,83 @@
+-- v_acquirablefile
+CREATE OR REPLACE VIEW v_acquirablefile AS (
+    SELECT a.id           AS acquirable_id,
+           a.name         AS acquirable_name,
+           a.slug         AS acquirable_slug,
+           f.id           AS id,
+           f.datetime     AS datetime,
+           f.file         AS file,
+           f.create_date  AS create_date,
+           f.process_date AS process_date
+    FROM acquirablefile f
+    LEFT JOIN acquirable a ON a.id = f.acquirable_id
+);
+
+-- v_product
+CREATE OR REPLACE VIEW v_product AS (
+    WITH tags_by_product AS (
+		SELECT product_id         AS product_id,
+               array_agg(tag_id)  AS tags
+	    FROM product_tags
+	    GROUP BY product_id
+	)
+	SELECT a.id                              AS id,
+           a.slug                            AS slug,
+           CONCAT(
+               UPPER(s.slug), ' ', 
+               (CASE WHEN LENGTH(a.label) > 1
+                     THEN CONCAT(a.label, ' ')
+                     ELSE ''
+                END), 
+                p.name, ' ',
+                a.temporal_resolution/60/60, 'hr'
+           )                                 AS name,
+           a.label                           AS label,
+           a.temporal_resolution             AS temporal_resolution,
+           a.temporal_duration               AS temporal_duration,
+           a.dss_fpart                       AS dss_fpart,
+           a.description                     AS description,
+           a.suite_id                        AS suite_id,
+           s.name                            AS suite,
+           COALESCE(t.tags, '{}')            AS tags,
+           p.id                              AS parameter_id,
+           p.name                            AS parameter,
+           u.id                              AS unit_id,
+           u.name                            AS unit,
+           pf.after                          AS after,
+           pf.before                         AS before,
+           COALESCE(pf.productfile_count, 0) AS productfile_count
+	FROM product a
+	JOIN unit u ON u.id = a.unit_id
+	JOIN parameter p ON p.id = a.parameter_id
+    JOIN suite s ON s.id = a.suite_id
+	LEFT JOIN tags_by_product t ON t.product_id = a.id
+    LEFT JOIN (
+        SELECT product_id    AS product_id,
+                COUNT(id)     AS productfile_count,
+                MIN(datetime) AS after,
+                MAX(datetime) AS before
+        FROM productfile
+        GROUP BY product_id
+    ) AS pf ON pf.product_id = a.id
+    WHERE NOT a.deleted
+    order by name
+);
+
+-- v_productfile
+CREATE OR REPLACE VIEW v_productfile AS (
+    SELECT p.id           AS product_id,
+           p.name         AS product_name,
+           p.slug         AS product_slug,
+           f.id           AS id,
+           f.datetime     AS datetime,
+           f.file         AS file,
+           f.version      AS version
+    FROM productfile f
+    LEFT JOIN v_product p ON p.id = f.product_id
+);
+
+GRANT SELECT ON
+    v_acquirablefile,
+    v_product,
+    v_productfile
+TO cumulus_reader;

api/_sql/R__05_views_downloads.sql

@@ -0,0 +1,102 @@
+CREATE OR REPLACE VIEW v_download AS (
+    SELECT d.id AS id,
+        d.datetime_start AS datetime_start,
+        d.datetime_end AS datetime_end,
+        d.progress AS progress,
+        d.file AS file,
+        d.processing_start AS processing_start,
+        d.processing_end AS processing_end,
+        d.status_id AS status_id,
+        d.watershed_id AS watershed_id,
+        d.profile_id AS profile_id,
+        w.slug         AS watershed_slug,
+        w.name         AS watershed_name,
+        s.name AS status,
+        dp.product_id AS product_id
+    FROM download d
+        INNER JOIN download_status s ON d.status_id = s.id
+        INNER JOIN watershed w on w.id = d.watershed_id
+        INNER JOIN (
+            SELECT array_agg(product_id) as product_id,
+                    download_id
+            FROM download_product
+            GROUP BY download_id
+        ) dp ON d.id = dp.download_id
+        ORDER BY d.processing_start DESC
+);
+
+-- v_download_request VIEW
+CREATE OR REPLACE VIEW v_download_request AS (
+    WITH download_products AS (
+        SELECT dp.download_id,
+            dp.product_id,
+            d.datetime_start,
+            d.datetime_end
+        FROM download d
+        JOIN download_product dp ON dp.download_id = d.id
+    )
+    SELECT dss.download_id,
+        dss.product_id,
+        dss.datetime_start,
+        dss.datetime_end,
+        dss.key,
+        dss.bucket,
+        dss.dss_datatype,
+        dss.dss_cpart,
+        CASE
+            WHEN dss.dss_datatype = 'INST-VAL'::text AND date_part('hour'::text, dss.datetime_dss_dpart) = 0::double precision 
+                AND date_part('minute'::text, dss.datetime_dss_dpart) = 0::double precision 
+            THEN to_char(dss.datetime_dss_dpart - '1 day'::interval, 'DDMONYYYY:24MI'::text)
+            ELSE COALESCE(to_char(dss.datetime_dss_dpart, 'DDMONYYYY:HH24MI'::text), ''::text)
+        END AS dss_dpart,
+        CASE
+            WHEN date_part('hour'::text, dss.datetime_dss_epart) = 0::double precision 
+                AND date_part('minute'::text, dss.datetime_dss_dpart) = 0::double precision 
+            THEN to_char(dss.datetime_dss_epart - '1 day'::interval, 'DDMONYYYY:24MI'::text)
+            ELSE COALESCE(to_char(dss.datetime_dss_epart, 'DDMONYYYY:HH24MI'::text), ''::text)
+        END AS dss_epart,
+        dss.dss_fpart,
+        dss.dss_unit,
+        dss.forecast_version
+    FROM (
+        SELECT dp.download_id,
+               dp.product_id,
+               dp.datetime_start,
+               dp.datetime_end,
+               f.file AS key,
+               (SELECT config.config_value FROM config WHERE config.config_name::text = 'write_to_bucket'::text) AS bucket,
+               CASE
+                   WHEN p.temporal_duration = 0 THEN 'INST-VAL'::text
+                   ELSE 'PER-CUM'::text
+               END AS dss_datatype,
+               CASE
+                   WHEN p.temporal_duration = 0 THEN f.datetime
+                   ELSE f.datetime - p.temporal_duration::double precision * '00:00:01'::interval
+               END AS datetime_dss_dpart,
+               CASE
+                   WHEN p.temporal_duration = 0 THEN NULL::timestamp with time zone
+                   ELSE f.datetime
+               END AS datetime_dss_epart,
+               p.dss_fpart,
+               u.name AS dss_unit,
+               a.name AS dss_cpart,
+               f.version AS forecast_version
+        FROM productfile f
+        JOIN download_products dp ON dp.product_id = f.product_id
+        JOIN product p ON f.product_id = p.id
+        JOIN unit u ON p.unit_id = u.id
+        JOIN parameter a ON a.id = p.parameter_id
+        -- observed data will use the file datetime
+        WHERE (date_part('year', f.version) = '1111' AND f.datetime >= dp.datetime_start AND f.datetime <= dp.datetime_end)
+        -- forecast data with an end date < now (looking at forecasts in the past)
+        OR (dp.datetime_end < now() AND date_part('year', f.version) != '1111' AND f.version between dp.datetime_end - interval '24 hours' and dp.datetime_end)
+        -- forecast data with an end date >= now (looking at current latest forecasts)
+        OR (dp.datetime_end >= now() AND date_part('year', f.version) != '1111' AND f.version between now() - interval '18 hours' and now())
+        ORDER BY f.product_id, f.version, f.datetime
+    ) dss
+);
+
+GRANT SELECT ON
+    v_download,
+    v_download_request
+TO cumulus_reader;