From e6773bd9fb49df14e9d375b245bdfde4e8b15fb6 Mon Sep 17 00:00:00 2001
From: jzonthemtn <jeff.zemerick@mtnfog.com>
Date: Wed, 18 Dec 2024 11:30:53 -0500
Subject: [PATCH] Adding routing.

Signed-off-by: jzonthemtn <jeff.zemerick@mtnfog.com>
---
 osi/terraform/get-indexed-queries.sh | 17 +++++++++++++++++
 osi/terraform/main.tf                | 26 +++++++++++++++++++++++++-
 osi/terraform/send-event.sh          |  4 ++--
 osi/terraform/send-query.sh          | 14 ++++++++++++++
 4 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100755 osi/terraform/get-indexed-queries.sh
 create mode 100755 osi/terraform/send-query.sh

diff --git a/osi/terraform/get-indexed-queries.sh b/osi/terraform/get-indexed-queries.sh
new file mode 100755
index 0000000..56a10d2
--- /dev/null
+++ b/osi/terraform/get-indexed-queries.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com"
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \
+  -X GET \
+  --region us-east-1 \
+  --service es \
+  --profile mtnfog
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \
+  -X GET \
+  --region us-east-1 \
+  --service es \
+  --profile mtnfog | jq
diff --git a/osi/terraform/main.tf b/osi/terraform/main.tf
index 9a9255c..88eac45 100644
--- a/osi/terraform/main.tf
+++ b/osi/terraform/main.tf
@@ -134,11 +134,14 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" {
             ubi-pipeline:
               source:
                 http:
-                  path: "/ubi_events"
+                  path: "/ubi"
               processor:
                 - date:
                     from_time_received: true
                     destination: "@timestamp"
+              route:
+                - ubi-events: '/type == "event"'
+                - ubi-queries: '/type == "query"'
               sink:
                 - opensearch:
                     hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
@@ -146,6 +149,7 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" {
                     aws:
                       sts_role_arn: "${aws_iam_role.ubi.arn}"   
                       region: "${data.aws_region.current.name}"
+                    routes: [ubi-events]
                 - s3:
                     aws:
                       sts_role_arn: "${aws_iam_role.ubi.arn}"
@@ -157,6 +161,26 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" {
                       event_collect_timeout: "60s"
                     codec:
                       ndjson:
+                    routes: [ubi-events]
+                - opensearch:
+                    hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
+                    index: "ubi_queries"
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"   
+                      region: "${data.aws_region.current.name}"
+                    routes: [ubi-queries]
+                - s3:
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"
+                      region: "${data.aws_region.current.name}"
+                    bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}"
+                    object_key:
+                      path_prefix: ubi_queries/
+                    threshold:
+                      event_collect_timeout: "60s"
+                    codec:
+                      ndjson:
+                    routes: [ubi-queries]                    
         EOT
   max_units                   = 1
   min_units                   = 1
diff --git a/osi/terraform/send-event.sh b/osi/terraform/send-event.sh
index d652139..14552d6 100755
--- a/osi/terraform/send-event.sh
+++ b/osi/terraform/send-event.sh
@@ -10,5 +10,5 @@ awscurl \
 	--profile mtnfog \
 	-X POST \
 	-H "Content-Type: application/json" \
-	-d '[{"action_name": "click", "query_id": "00112233-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \
-	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi_events
+	-d '[{"type": "event", "action_name": "click", "query_id": "00112233-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi
diff --git a/osi/terraform/send-query.sh b/osi/terraform/send-query.sh
new file mode 100755
index 0000000..a7cdd52
--- /dev/null
+++ b/osi/terraform/send-query.sh
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+# pip install awscurl
+
+OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com"
+
+awscurl \
+	--service osis \
+	--region us-east-1 \
+	--profile mtnfog \
+	-X POST \
+	-H "Content-Type: application/json" \
+	-d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi