From 0b73fa90160d93c9109f0515efcd39d60db1724c Mon Sep 17 00:00:00 2001 From: Andres Gregori Altamirano Date: Thu, 11 Jul 2024 19:39:38 +0000 Subject: [PATCH] Added notes on session table, changed BQML logic to optional --- dashboards/arima.dashboard.lookml | 6 ++-- dashboards/avbb.dashboard.lookml | 6 ++-- ...performance_metrics_model.dashboard.lookml | 6 ++-- dashboards/propensity_model.dashboard.lookml | 6 ++-- manifest.lkml | 6 +++- marketplace.json | 4 +++ models/ga4.model.lkml | 1 + .../arima_event_model/forecasting.view.lkml | 5 +-- .../arima_event_model/model_arima.view.lkml | 14 +++++--- .../training_input_arima.view.lkml | 3 +- views/bqml/avbb/training_data_avbb.view.lkml | 5 +-- .../future_input.view.lkml | 2 +- .../incremental_prediction.view.lkml | 10 ++++-- .../purchase_propensity/predictions.view.lkml | 35 +++++++++++-------- .../testing_input.view.lkml | 2 +- .../training_input.view.lkml | 3 +- .../sessions/session_event_packing.view.lkml | 6 ++++ views/sessions/session_facts.view.lkml | 6 ++++ .../session_list_with_event_history.view.lkml | 12 ++++++- views/sessions/session_tags.view.lkml | 7 ++++ 20 files changed, 102 insertions(+), 43 deletions(-) diff --git a/dashboards/arima.dashboard.lookml b/dashboards/arima.dashboard.lookml index 773a72e..15a1088 100644 --- a/dashboards/arima.dashboard.lookml +++ b/dashboards/arima.dashboard.lookml @@ -68,11 +68,11 @@ svg-icon\" viewBox=\"0 0 20 20\">\n\n\nPropensity Performance\n\n\n\n\n\nARIMA Event Model\n\n\n\n\n\nAggregated Value Based Bidding\n\n" @@ -560,4 +560,4 @@ model: ga4 explore: sessions listens_to_filters: [] - field: events.event_name \ No newline at end of file + field: events.event_name diff --git a/dashboards/avbb.dashboard.lookml b/dashboards/avbb.dashboard.lookml index 38b94e1..becd56f 100644 --- a/dashboards/avbb.dashboard.lookml +++ b/dashboards/avbb.dashboard.lookml @@ -69,11 +69,11 @@ svg-icon\" viewBox=\"0 0 20 20\">\n\n\nPropensity Performance\n\n\n\n\n\nARIMA Event Model\n\n\n\n\n\nAggregated Value Based Bidding\n\n" @@ -454,4 +454,4 @@ model: ga4 explore: category_attribution listens_to_filters: [] - field: category_attribution.processed_input \ No newline at end of file + field: category_attribution.processed_input diff --git a/dashboards/performance_metrics_model.dashboard.lookml b/dashboards/performance_metrics_model.dashboard.lookml index 82c593f..fd25172 100644 --- a/dashboards/performance_metrics_model.dashboard.lookml +++ b/dashboards/performance_metrics_model.dashboard.lookml @@ -66,11 +66,11 @@ svg-icon\" viewBox=\"0 0 20 20\">\n\n\nPropensity Performance\n\n\n\n\n\nARIMA Event Model\n\n\n\n\n\nAggregated Value Based Bidding\n\n" @@ -385,4 +385,4 @@ model: ga4 explore: sessions listens_to_filters: [] - field: sessions.session_date \ No newline at end of file + field: sessions.session_date diff --git a/dashboards/propensity_model.dashboard.lookml b/dashboards/propensity_model.dashboard.lookml index 209697b..59e1729 100644 --- a/dashboards/propensity_model.dashboard.lookml +++ b/dashboards/propensity_model.dashboard.lookml @@ -66,11 +66,11 @@ svg-icon\" viewBox=\"0 0 20 20\">\n\n\nPropensity Performance\n\n\n\n\n\nARIMA Event Model\n\n\n\n\n\nAggregated Value Based Bidding\n\n" @@ -441,4 +441,4 @@ model: ga4 explore: sessions listens_to_filters: [] - field: sessions.session_date \ No newline at end of file + field: sessions.session_date diff --git a/manifest.lkml b/manifest.lkml index 72593b8..efdc85c 100644 --- a/manifest.lkml +++ b/manifest.lkml @@ -2,7 +2,7 @@ project_name: "ga_four" ## Connection Constants: constant: GA4_CONNECTION { - value: "" + value: "bq-looker-marketplace" export: override_required } @@ -23,6 +23,10 @@ constant: model_step_prediction { value: "60" #export: override_optional } +constant: BQML_PARAMETER { + value: "Yes" + export: override_optional +} constant: GA4_BQML_train_months { value: "12" diff --git a/marketplace.json b/marketplace.json index f138f36..21c7808 100644 --- a/marketplace.json +++ b/marketplace.json @@ -19,6 +19,10 @@ "EVENT_COUNT": { "label": "Event Count Limit" } + }, + "BQML_PARAMETER": { + "label": "BQML Models Included" + } }, "models": [ { diff --git a/models/ga4.model.lkml b/models/ga4.model.lkml index bde80f9..83fe876 100644 --- a/models/ga4.model.lkml +++ b/models/ga4.model.lkml @@ -21,4 +21,5 @@ datagroup: ga4_attribution_channel { sql_trigger: SELECT 1 ;; } + persist_with: ga4_main_datagroup diff --git a/views/bqml/arima_event_model/forecasting.view.lkml b/views/bqml/arima_event_model/forecasting.view.lkml index bbf42e8..631b7bf 100644 --- a/views/bqml/arima_event_model/forecasting.view.lkml +++ b/views/bqml/arima_event_model/forecasting.view.lkml @@ -2,8 +2,9 @@ include: "/views/bqml/*/*.view.lkml" view: forecasting { derived_table: { sql_trigger_value: ${model_arima.SQL_TABLE_NAME} ;; - sql: SELECT * FROM ML.EXPLAIN_FORECAST(MODEL ${model_arima.SQL_TABLE_NAME}, - STRUCT(60 AS horizon, 0.8 AS confidence_level)) ;; + sql: + SELECT * FROM ML.EXPLAIN_FORECAST(MODEL ${model_arima.SQL_TABLE_NAME}, + STRUCT(60 AS horizon, 0.8 AS confidence_level));; } dimension: forecast_timestamp { diff --git a/views/bqml/arima_event_model/model_arima.view.lkml b/views/bqml/arima_event_model/model_arima.view.lkml index 2503780..548dd52 100644 --- a/views/bqml/arima_event_model/model_arima.view.lkml +++ b/views/bqml/arima_event_model/model_arima.view.lkml @@ -4,7 +4,8 @@ view: model_arima { #extension: required derived_table: { sql_trigger_value: ${training_input_arima.SQL_TABLE_NAME};; - sql_create: CREATE OR REPLACE MODEL ${SQL_TABLE_NAME} + sql_create: + CREATE OR REPLACE MODEL ${SQL_TABLE_NAME} OPTIONS( MODEL_TYPE='ARIMA_PLUS', time_series_timestamp_col='sessions_session_date', @@ -12,7 +13,8 @@ view: model_arima { time_series_id_col='events_event_name', auto_arima=true) AS SELECT * - FROM ${training_input_arima.SQL_TABLE_NAME};; + FROM ${training_input_arima.SQL_TABLE_NAME} +;; } dimension: ts { type: string @@ -35,7 +37,9 @@ view: model_arima { #explore: forecasting {} view: optimal_model_coeff{ derived_table: { - sql: SELECT * FROM ML.ARIMA_COEFFICIENTS(MODEL ${model_arima.SQL_TABLE_NAME});; + sql: + SELECT * FROM ML.ARIMA_COEFFICIENTS(MODEL ${model_arima.SQL_TABLE_NAME}) +;; } dimension: ar_coefficients {type:number sql:${TABLE}.ar_coefficients;;} dimension: ma_coefficients {type:number sql:${TABLE}.ma_coefficients;;} @@ -43,7 +47,9 @@ view: model_arima { } view: model_evaluation { derived_table: { - sql: SELECT * FROM ML.ARIMA_EVALUATE(MODEL ${model_arima.SQL_TABLE_NAME});; + sql: + SELECT * FROM ML.ARIMA_EVALUATE(MODEL ${model_arima.SQL_TABLE_NAME}) +;; } dimension: events_event_name {type:string sql:${TABLE}.events_event_name ;;} dimension: has_drift {type:string sql:${TABLE}.has_drift ;;} diff --git a/views/bqml/arima_event_model/training_input_arima.view.lkml b/views/bqml/arima_event_model/training_input_arima.view.lkml index 104e532..45f51ed 100644 --- a/views/bqml/arima_event_model/training_input_arima.view.lkml +++ b/views/bqml/arima_event_model/training_input_arima.view.lkml @@ -1,7 +1,8 @@ view: training_input_arima { derived_table: { sql_trigger_value: ${incremental_prediction.SQL_TABLE_NAME} ;; - sql:SELECT + sql: + SELECT (DATE(TIMESTAMP_MICROS(events.event_timestamp) )) AS sessions_session_date, events.event_name AS events_event_name, COUNT(DISTINCT sessions.sl_key ) AS sum_of_session_event_count diff --git a/views/bqml/avbb/training_data_avbb.view.lkml b/views/bqml/avbb/training_data_avbb.view.lkml index 3788e28..1514a75 100644 --- a/views/bqml/avbb/training_data_avbb.view.lkml +++ b/views/bqml/avbb/training_data_avbb.view.lkml @@ -3,7 +3,8 @@ include: "/views/*/*.view.lkml" view: training_data_avbb { derived_table: { sql_trigger_value: ${forecasting.SQL_TABLE_NAME} ;; - sql:SELECT sessions.sl_key, + sql: + SELECT sessions.sl_key, session_attribution.source, session_attribution.medium, session_attribution.campaign, @@ -14,7 +15,7 @@ view: training_data_avbb { SUM(session_data.session_page_view_count) as session_count, COALESCE(SUM(event_data[SAFE_OFFSET(0)].user_ltv.revenue),0.0) as label FROM ${sessions.SQL_TABLE_NAME} as sessions - LEFT JOIN UNNEST(sessions.event_data) as events with offset as event_row GROUP BY 1,2,3,4,5,6,7 ;; + LEFT JOIN UNNEST(sessions.event_data) as events with offset as event_row GROUP BY 1,2,3,4,5,6,7;; } dimension: sl_key { hidden: yes diff --git a/views/bqml/purchase_propensity/future_input.view.lkml b/views/bqml/purchase_propensity/future_input.view.lkml index 0653a66..d9d8692 100644 --- a/views/bqml/purchase_propensity/future_input.view.lkml +++ b/views/bqml/purchase_propensity/future_input.view.lkml @@ -212,6 +212,6 @@ view: future_input { SELECT * EXCEPT(will_purchase_in_future) FROM - user_model);; + CASE WHEN @{BQML_PARAMETER}='Yes' THEN user_model ELSE (SELECT 1,"1" as will_purchase_in_future) END;; } } diff --git a/views/bqml/purchase_propensity/incremental_prediction.view.lkml b/views/bqml/purchase_propensity/incremental_prediction.view.lkml index 5ebf769..522728e 100644 --- a/views/bqml/purchase_propensity/incremental_prediction.view.lkml +++ b/views/bqml/purchase_propensity/incremental_prediction.view.lkml @@ -6,6 +6,7 @@ include: "/views/*.view.lkml" explore: pred_history {} view: pred_history { derived_table: { + sql_trigger_value:CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END;; explore_source: sessions { column: pred_probability_bucket { field: future_purchase_prediction.pred_probability_bucket } column: total_purchase_revenue_usd { field: events.total_purchase_revenue_usd } @@ -56,7 +57,7 @@ explore: incremental_prediction {hidden:yes} view: incremental_prediction { derived_table: { #datagroup_trigger: bqml_datagroup - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; + sql_trigger_value:CASE WHEN @{BQML_PARAMETER}='Yes' AND ${pred_history.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END;; create_process: { sql_step: CREATE TABLE IF NOT EXISTS ${SQL_TABLE_NAME} ( @@ -64,13 +65,16 @@ view: incremental_prediction { total_purchase_revenue_usd FLOAT64, count INT64, week DATE - );; + ) +;; sql_step: + Insert into ${SQL_TABLE_NAME} SELECT pred_probability_bucket,total_purchase_revenue_usd,count, week FROM ${pred_history.SQL_TABLE_NAME} - where week not in (select distinct week from ${SQL_TABLE_NAME});; + where week not in (select distinct week from ${SQL_TABLE_NAME}) +;; } } dimension: week { diff --git a/views/bqml/purchase_propensity/predictions.view.lkml b/views/bqml/purchase_propensity/predictions.view.lkml index 05af958..2e7b338 100644 --- a/views/bqml/purchase_propensity/predictions.view.lkml +++ b/views/bqml/purchase_propensity/predictions.view.lkml @@ -6,7 +6,7 @@ include: "/views/sessions/*.view" view: future_purchase_model { derived_table: { - sql_trigger_value: ${future_input.SQL_TABLE_NAME} ;; + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_input.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; sql_create: CREATE OR REPLACE MODEL ${SQL_TABLE_NAME} OPTIONS( @@ -41,10 +41,12 @@ explore: feature_importance {hidden:yes} # VIEWS: view: future_purchase_model_evaluation { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: SELECT * FROM ml.EVALUATE( + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + SELECT * FROM ml.EVALUATE( MODEL ${future_purchase_model.SQL_TABLE_NAME}, - (SELECT * FROM ${testing_input.SQL_TABLE_NAME}));; + (SELECT * FROM ${testing_input.SQL_TABLE_NAME})) +;; } dimension: recall {type: number value_format_name:percent_2} dimension: accuracy {type: number value_format_name:percent_2} @@ -55,8 +57,9 @@ view: future_purchase_model_evaluation { view: roc_curve { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: SELECT * FROM ml.ROC_CURVE( + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + SELECT * FROM ml.ROC_CURVE( MODEL ${future_purchase_model.SQL_TABLE_NAME}, (SELECT * FROM ${testing_input.SQL_TABLE_NAME}));; } @@ -101,8 +104,9 @@ view: roc_curve { view: confusion_matrix { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: SELECT Expected_label,_0 as Predicted_0,_1 as Predicted_1 FROM ml.confusion_matrix( + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + SELECT Expected_label,_0 as Predicted_0,_1 as Predicted_1 FROM ml.confusion_matrix( MODEL ${future_purchase_model.SQL_TABLE_NAME}, (SELECT * FROM ${testing_input.SQL_TABLE_NAME}));; } @@ -113,8 +117,9 @@ view: confusion_matrix { view: future_purchase_model_training_info { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: SELECT * FROM ml.TRAINING_INFO(MODEL ${future_purchase_model.SQL_TABLE_NAME});; + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + SELECT * FROM ml.TRAINING_INFO(MODEL ${future_purchase_model.SQL_TABLE_NAME});; } dimension: training_run {type: number} dimension: iteration {type: number} @@ -146,8 +151,9 @@ view: future_purchase_model_training_info { view: feature_importance { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: SELECT + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + SELECT * FROM ML.GLOBAL_EXPLAIN(MODEL ${future_purchase_model.SQL_TABLE_NAME});; @@ -160,8 +166,9 @@ view: feature_importance { explore: future_purchase_prediction {hidden:yes} view: future_purchase_prediction { derived_table: { - sql_trigger_value: ${future_purchase_model.SQL_TABLE_NAME} ;; - sql: select + sql_trigger_value: CASE WHEN @{BQML_PARAMETER}='Yes' AND ${future_purchase_model.SQL_TABLE_NAME} THEN TRUE ELSE FALSE END ;; + sql: + select pred.*, predicted_will_purchase_in_future_probs_unnest.prob as pred_probability from (SELECT * FROM ml.PREDICT( diff --git a/views/bqml/purchase_propensity/testing_input.view.lkml b/views/bqml/purchase_propensity/testing_input.view.lkml index f96d87b..63e6923 100644 --- a/views/bqml/purchase_propensity/testing_input.view.lkml +++ b/views/bqml/purchase_propensity/testing_input.view.lkml @@ -210,6 +210,6 @@ view: testing_input { SELECT * FROM - user_model);; + CASE WHEN @{BQML_PARAMETER}='Yes' THEN user_model ELSE (SELECT 1) END;; } } diff --git a/views/bqml/purchase_propensity/training_input.view.lkml b/views/bqml/purchase_propensity/training_input.view.lkml index f3fe331..ebe4337 100644 --- a/views/bqml/purchase_propensity/training_input.view.lkml +++ b/views/bqml/purchase_propensity/training_input.view.lkml @@ -2,6 +2,7 @@ include: "/views/sessions/*.view.lkml" include: "/views/*/*.view.lkml" view: training_input { derived_table: { + #datagroup_trigger: bqml_datagroup sql_trigger_value: ${sessions.SQL_TABLE_NAME} ;; sql: @@ -211,6 +212,6 @@ view: training_input { SELECT * FROM - user_model;; + CASE WHEN @{BQML_PARAMETER}='Yes' THEN user_model ELSE (SELECT 1) END;; } } diff --git a/views/sessions/session_event_packing.view.lkml b/views/sessions/session_event_packing.view.lkml index f4d2b3f..01b632f 100644 --- a/views/sessions/session_event_packing.view.lkml +++ b/views/sessions/session_event_packing.view.lkml @@ -1,4 +1,10 @@ include: "/views/sessions/*.view.lkml" +########################### +# 04: SESSION EVENT PACKING +# In this query we get all of the items that +# were nested and aggregate them into one. This is then used in the events table. +# +############################ view: session_event_packing { derived_table:{ sql_trigger_value: ${session_facts.SQL_TABLE_NAME} ;; diff --git a/views/sessions/session_facts.view.lkml b/views/sessions/session_facts.view.lkml index 5ec7161..81c7fcd 100644 --- a/views/sessions/session_facts.view.lkml +++ b/views/sessions/session_facts.view.lkml @@ -1,4 +1,10 @@ include: "/views/sessions/*.view.lkml" +########################### +# 03: SESSION FACTS +# This PDT gets basic facts for the session like when it started, +# how many pages it viewed, and the length in minutes. +# Primary Key is always sl_key +############################ view: session_facts{ derived_table: { #datagroup_trigger: ga4_main_datagroup diff --git a/views/sessions/session_list_with_event_history.view.lkml b/views/sessions/session_list_with_event_history.view.lkml index f750484..579e209 100644 --- a/views/sessions/session_list_with_event_history.view.lkml +++ b/views/sessions/session_list_with_event_history.view.lkml @@ -1,6 +1,11 @@ include: "/views/event_data_dimensions/event_funnel.view" include: "/views/event_data_dimensions/page_funnel.view" - +########################### +# 01: SESSION LIST WITH EVENT HISTORY +# This is the backbone of the block. +# In this query, we parse the date from the table name +# And sessionalize all the events with the sl_key +############################ view: session_list_with_event_history { derived_table: { datagroup_trigger: ga4_main_datagroup @@ -58,4 +63,9 @@ view: session_list_with_event_history { hidden: yes sql: ${TABLE}.session_date;; } + parameter: bqm_enabled { + type: unquoted + default_value: "@{BQML_PARAMETER}" + + } } diff --git a/views/sessions/session_tags.view.lkml b/views/sessions/session_tags.view.lkml index 857808c..2dc72d4 100644 --- a/views/sessions/session_tags.view.lkml +++ b/views/sessions/session_tags.view.lkml @@ -1,4 +1,11 @@ include: "/views/sessions/*.view.lkml" +########################### +# 02: SESSION TAGS +# This is the backbone of the block. +# In this PDT we tag all users by medium, +# source, campaign, and page referrer. +# A user once tagged, it won't change. +############################ view: session_tags{ derived_table:{ increment_key: "session_date"