Terraform for OpenSearch Ingestion pipeline for UBI

osi/create-plugin-package.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -e
+
+. stack.properties
+
+# This script is just here for an example.
+# AWS OpenSearch does not (yet) support ActionPlugins.
+
+# https://docs.aws.amazon.com/opensearch-service/latest/developerguide/custom-plugins.html
+# Note that custom plugins cannot implement ActionPlugin.
+
+FILE_NAME="search-quality-evaluation-plugin-0.0.1.zip"
+
+wget -O ${FILE_NAME} https://github.com/o19s/opensearch-search-quality-evaluation/releases/download/0.0.1/${FILE_NAME}
+
+aws s3 cp ${FILE_NAME} s3://ubi-queries-events-sink/${FILE_NAME} \
+    --profile ${AWS_PROFILE} \
+    --region ${AWS_REGION}
+
+aws opensearch create-package \
+    --region ${AWS_REGION} \
+    --profile ${AWS_PROFILE} \
+    --package-name search-eval-framework \
+    --package-type ZIP-PLUGIN \
+    --package-source S3BucketName=ubi-queries-events-sink,S3Key=search-quality-evaluation-plugin-0.0.1.zip \
+    --engine-version OpenSearch_2.17
+
+# aws opensearch describe-packages \
+#     --region ${AWS_REGION} \
+#     --profile ${AWS_PROFILE} \
+#     --filters '[{"Name": "PackageType","Value": ["ZIP-PLUGIN"]}, {"Name": "PackageName","Value": ["search-eval-framework"]}]'
+
+# when done, grab the package id and put into command:
+
+# PACKAGE_ID="pkg-b618759e2c2d03c7b9934b214ce6d09fcfaa8547"
+
+# aws opensearch associate-package \
+#     --region ${AWS_REGION} \
+#     --domain-name osi-ubi-domain \
+#     --profile ${AWS_PROFILE}
+#     --package-id ${PACKAGE_ID}
+
+
+# aws opensearch list-packages-for-domain
+#     --domain-name osi-ubi-domain \
+#     --region $REGION \
+#     --profile ${AWS_PROFILE}

osi/delete-indexes.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+. stack.properties
+
+OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .`
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/ubi_events,ubi_queries" \
+  -X DELETE \
+  --region ${AWS_REGION} \
+  --service es \
+  --profile ${AWS_PROFILE}

osi/get-indexed-events.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+. stack.properties
+
+OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .`
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \
+  -X GET \
+  --region ${AWS_REGION} \
+  --service es \
+  --profile ${AWS_PROFILE}
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/ubi_events/_search" \
+  -X GET \
+  --region ${AWS_REGION} \
+  --service es \
+  --profile ${AWS_PROFILE} | jq

osi/get-indexed-queries.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+. stack.properties
+
+OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .`
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \
+  -X GET \
+  --region ${AWS_REGION} \
+  --service es \
+  --profile ${AWS_PROFILE}
+
+awscurl \
+  "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \
+  -X GET \
+  --region ${AWS_REGION} \
+  --service es \
+  --profile ${AWS_PROFILE} | jq

osi/main.tf

@@ -0,0 +1,203 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.36"
+    }
+  }
+
+  required_version = ">= 1.2.0"
+}
+
+provider "aws" {
+  region  = "us-east-1"
+  profile = "mtnfog"
+}
+
+data "aws_region" "current" {}
+data "aws_caller_identity" "current" {}
+
+locals {
+  account_id = data.aws_caller_identity.current.account_id
+}
+
+output "ingest_endpoint_url" {
+  value = tolist(aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls)[0]
+}
+
+resource "aws_iam_role" "ubi" {
+  name = "ubiosisrole"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "osis-pipelines.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+data "aws_iam_policy_document" "access_policy" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["${aws_iam_role.ubi.arn}"]
+    }
+
+    actions = ["es:*"]
+  }
+}
+
+resource "aws_opensearch_domain" "opensearch_ubi" {
+
+  domain_name    = "osi-ubi-domain"
+  engine_version = "OpenSearch_2.17"
+
+  cluster_config {
+    instance_type = "t3.small.search"
+  }
+
+  encrypt_at_rest {
+    enabled = true
+  }
+
+  domain_endpoint_options {
+    enforce_https       = true
+    tls_security_policy = "Policy-Min-TLS-1-2-2019-07"
+  }
+
+  node_to_node_encryption {
+    enabled = true
+  }
+
+  ebs_options {
+    ebs_enabled = true
+    volume_size = 10
+  }
+
+  #access_policies = data.aws_iam_policy_document.access_policy.json
+}
+
+resource "aws_iam_policy" "ubi" {
+  name        = "osis_role_policy"
+  description = "Policy for OSIS pipeline role"
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Action   = ["es:DescribeDomain"]
+        Effect   = "Allow"
+        Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/*"
+      },
+      {
+        Action   = ["es:ESHttp*"]
+        Effect   = "Allow"
+        Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/osi-ubi-domain/*"
+      },
+      {
+        Action   = ["s3:PutObject"]
+        Effect   = "Allow"
+        Resource = "arn:aws:s3:::${aws_s3_bucket.ubi_queries_events_bucket.id}/*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "ubi" {
+  role       = aws_iam_role.ubi.name
+  policy_arn = aws_iam_policy.ubi.arn
+}
+
+resource "aws_cloudwatch_log_group" "ubi" {
+  name              = "/aws/vendedlogs/OpenSearchIngestion/ubi-pipeline"
+  retention_in_days = 14
+  tags = {
+    Name = "UBI OSIS Pipeline Log Group"
+  }
+}
+
+resource "aws_s3_bucket" "ubi_queries_events_bucket" {
+  bucket = "ubi-queries-events-sink"
+}
+
+resource "aws_osis_pipeline" "ubi_events_pipeline" {
+  pipeline_name               = "ubi-pipeline"
+  pipeline_configuration_body = <<-EOT
+            version: "2"
+            ubi-pipeline:
+              source:
+                http:
+                  path: "/ubi"
+              processor:
+                - date:
+                    from_time_received: true
+                    destination: "@timestamp"
+              route:
+                - ubi-events: '/type == "event"'
+                - ubi-queries: '/type == "query"'
+              sink:
+                - opensearch:
+                    hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
+                    index: "ubi_events"
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"   
+                      region: "${data.aws_region.current.name}"
+                    routes: [ubi-events]
+                - s3:
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"
+                      region: "${data.aws_region.current.name}"
+                    bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}"
+                    object_key:
+                      path_prefix: ubi_events/
+                    threshold:
+                      event_collect_timeout: "60s"
+                    codec:
+                      ndjson:
+                    routes: [ubi-events]
+                - opensearch:
+                    hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"]
+                    index: "ubi_queries"
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"   
+                      region: "${data.aws_region.current.name}"
+                    routes: [ubi-queries]
+                - s3:
+                    aws:
+                      sts_role_arn: "${aws_iam_role.ubi.arn}"
+                      region: "${data.aws_region.current.name}"
+                    bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}"
+                    object_key:
+                      path_prefix: ubi_queries/
+                    threshold:
+                      event_collect_timeout: "60s"
+                    codec:
+                      ndjson:
+                    routes: [ubi-queries]                    
+        EOT
+  max_units                   = 1
+  min_units                   = 1
+  log_publishing_options {
+    is_logging_enabled = true
+    cloudwatch_log_destination {
+      log_group = aws_cloudwatch_log_group.ubi.name
+    }
+  }
+  tags = {
+    Name = "UBI OpenSearch Ingestion Pipeline for UBI"
+  }
+}
+
+output "opensearch_domain_endpoint" {
+  value = aws_opensearch_domain.opensearch_ubi.endpoint
+}
+
+output "opensearch_ingest_pipeline_endpoint" {
+  value = aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls
+}

osi/requirements.txt

@@ -0,0 +1 @@
+awscurl==0.36

osi/send-event.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+. stack.properties
+
+OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]`
+
+awscurl \
+	--service osis \
+	--region ${AWS_REGION} \
+	--profile ${AWS_PROFILE} \
+	-X POST \
+	-H "Content-Type: application/json" \
+	-d '[{"type": "event", "action_name": "click", "query_id": "99999999-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi

osi/send-query.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+. stack.properties
+
+OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]`
+
+awscurl \
+	--service osis \
+	--region ${AWS_REGION} \
+	--profile ${AWS_PROFILE} \
+	-X POST \
+	-H "Content-Type: application/json" \
+	-d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \
+	https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi

osi/stack.properties

@@ -0,0 +1,5 @@
+# The name of the AWS profile.
+AWS_PROFILE=mtnfog
+
+# The AWS region.
+AWS_REGION=us-east-1