Adding routing.

Signed-off-by: jzonthemtn <[email protected]>
o19s · Dec 18, 2024 · e6773bd · e6773bd
1 parent 0459d46
commit e6773bd
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 3 deletions.
diff --git a/osi/terraform/get-indexed-queries.sh b/osi/terraform/get-indexed-queries.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com"
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \
+  -X GET \
+  --region us-east-1 \
+  --service es \
+  --profile mtnfog
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \
+  -X GET \
+  --region us-east-1 \
+  --service es \
+  --profile mtnfog | jq
diff --git a/osi/terraform/main.tf b/osi/terraform/main.tf
@@ -134,18 +134,22 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" {
             ubi-pipeline:
               source:
                 http:
-                  path: "/ubi_events"
+                  path: "/ubi"
               processor:
                 - date:
                     from_time_received: true
                     destination: "@timestamp"
+              route:
+                - ubi-events: '/type == "event"'
+                - ubi-queries: '/type == "query"'
               sink:
                 - opensearch:
                     hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
                     index: "ubi_events"
                     aws:
                       sts_role_arn: "${aws_iam_role.ubi.arn}"   
                       region: "${data.aws_region.current.name}"
+                    routes: [ubi-events]
                 - s3:
                     aws:
                       sts_role_arn: "${aws_iam_role.ubi.arn}"
@@ -157,6 +161,26 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" {
                       event_collect_timeout: "60s"
                     codec:
                       ndjson:
+                    routes: [ubi-events]
+                - opensearch:
+                    hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
+                    index: "ubi_queries"
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"   
+                      region: "${data.aws_region.current.name}"
+                    routes: [ubi-queries]
+                - s3:
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"
+                      region: "${data.aws_region.current.name}"
+                    bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}"
+                    object_key:
+                      path_prefix: ubi_queries/
+                    threshold:
+                      event_collect_timeout: "60s"
+                    codec:
+                      ndjson:
+                    routes: [ubi-queries]                    
         EOT
   max_units                   = 1
   min_units                   = 1

diff --git a/osi/terraform/send-event.sh b/osi/terraform/send-event.sh
@@ -10,5 +10,5 @@ awscurl \
 	--profile mtnfog \
 	-X POST \
 	-H "Content-Type: application/json" \
-	-d '[{"action_name": "click", "query_id": "00112233-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \
-	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi_events
+	-d '[{"type": "event", "action_name": "click", "query_id": "00112233-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi
diff --git a/osi/terraform/send-query.sh b/osi/terraform/send-query.sh
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+# pip install awscurl
+
+OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com"
+
+awscurl \
+	--service osis \
+	--region us-east-1 \
+	--profile mtnfog \
+	-X POST \
+	-H "Content-Type: application/json" \
+	-d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi