[sc-27082] Do the query text concatenation as CTE (#891)

* [sc-27082] Do the query text concatenation as CTE

* bump version

* fix the query

metaphor/redshift/access_event.py

@@ -5,12 +5,28 @@
 from asyncpg import Connection, Record
 
 REDSHIFT_USAGE_SQL_TEMPLATE = """
-WITH queries as (
+WITH queries AS (
     SELECT
-        sqt.*,
-        SUM(LEN(sqt.text)) OVER (PARTITION BY sqt.query) AS length
-    FROM stl_querytext as sqt
-    ORDER BY length DESC, sqt.sequence
+        *,
+        SUM(LEN(text)) OVER (PARTITION BY query) AS length
+    FROM
+        stl_querytext
+), filtered AS (
+    SELECT
+        q.query,
+        LISTAGG(
+            CASE
+                WHEN LEN(RTRIM(q.text)) = 0 THEN q.text
+                ELSE RTRIM(q.text)
+            END,
+            ''
+        ) WITHIN GROUP (ORDER BY q.sequence) AS querytxt
+    FROM
+        queries AS q
+    WHERE
+        q.length < 65536
+    GROUP BY
+        q.query
 )
 SELECT DISTINCT
     ss.userid,
@@ -19,23 +35,7 @@
     ss.rows,
     ss.bytes,
     ss.tbl,
-    REPLACE(
-        REPLACE(
-            LISTAGG(
-                CASE
-                    WHEN LEN(RTRIM(q.text)) = 0 THEN q.text
-                    ELSE RTRIM(q.text)
-                END,
-                ''
-            )
-            WITHIN GROUP (ORDER BY q.sequence)
-            OVER (
-                PARTITION BY q.userid, q.xid, q.pid, q.query
-            ),
-            '\r', ''
-        ),
-        '\\n', ''
-    ) AS querytxt,
+    q.querytxt,
     sti.database,
     sti.schema,
     sti.table,
@@ -45,36 +45,15 @@
 FROM stl_scan ss
     JOIN svv_table_info sti ON ss.tbl = sti.table_id
     JOIN stl_query sq ON ss.query = sq.query
-    JOIN queries q ON ss.query = q.query
+    JOIN filtered q ON ss.query = q.query
     JOIN svl_user_info sui ON sq.userid = sui.usesysid
 WHERE ss.starttime >= '{start_time}'
     AND ss.starttime < '{end_time}'
     AND sq.aborted = 0
-    AND q.length < 65536
-GROUP BY
-    ss.userid,
-    ss.query,
-    sui.usename,
-    ss.rows,
-    ss.bytes,
-    ss.tbl,
-    sti.database,
-    sti.schema,
-    sti.table,
-    sq.starttime,
-    sq.endtime,
-    sq.aborted,
-    ss.endtime,
-    q.text,
-    q.userid,
-    q.xid,
-    q.pid,
-    q.query,
-    q.sequence
 ORDER BY ss.endtime DESC;
 """
 """
-The condition `q.length < 65536` is because Redshift's LISTAGG method
+The condition `length < 65536` is because Redshift's LISTAGG method
 is unable to process the query if it is over 65535 characters long.
 See https://docs.aws.amazon.com/redshift/latest/dg/r_WF_LISTAGG.html#r_WF_LISTAGG-data-types
 """

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.26"
+version = "0.14.27"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]