From b141fbafd0025df7cbb060f4440a9e6c5de98a22 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 19 Dec 2024 11:55:32 -0500 Subject: [PATCH 1/4] Adding terraform scripts for UBI OSI pipeline. Signed-off-by: jzonthemtn --- osi/create-plugin-package.sh | 41 ++++++++ osi/delete-indexes.sh | 13 +++ osi/get-indexed-events.sh | 20 ++++ osi/get-indexed-queries.sh | 20 ++++ osi/main.tf | 195 +++++++++++++++++++++++++++++++++++ osi/send-event.sh | 15 +++ osi/send-query.sh | 15 +++ 7 files changed, 319 insertions(+) create mode 100755 osi/create-plugin-package.sh create mode 100755 osi/delete-indexes.sh create mode 100755 osi/get-indexed-events.sh create mode 100755 osi/get-indexed-queries.sh create mode 100644 osi/main.tf create mode 100755 osi/send-event.sh create mode 100755 osi/send-query.sh diff --git a/osi/create-plugin-package.sh b/osi/create-plugin-package.sh new file mode 100755 index 0000000..6584a14 --- /dev/null +++ b/osi/create-plugin-package.sh @@ -0,0 +1,41 @@ +#!/bin/bash -e + +# https://docs.aws.amazon.com/opensearch-service/latest/developerguide/custom-plugins.html +# Note that custom plugins cannot implement ActionPlugin. + +FILE_NAME="search-quality-evaluation-plugin-0.0.1.zip" + +wget -O ${FILE_NAME} https://github.com/o19s/opensearch-search-quality-evaluation/releases/download/0.0.1/${FILE_NAME} + +aws s3 cp ${FILE_NAME} s3://ubi-queries-events-sink/${FILE_NAME} \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + --region us-east-1 + +aws opensearch create-package \ + --region us-east-1 \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + --package-name search-eval-framework \ + --package-type ZIP-PLUGIN \ + --package-source S3BucketName=ubi-queries-events-sink,S3Key=search-quality-evaluation-plugin-0.0.1.zip \ + --engine-version OpenSearch_2.17 + +# aws opensearch describe-packages \ +# --region us-east-1 \ +# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ +# --filters '[{"Name": "PackageType","Value": ["ZIP-PLUGIN"]}, {"Name": "PackageName","Value": ["search-eval-framework"]}]' + +# when done, grab the package id and put into command: + +# PACKAGE_ID="pkg-b618759e2c2d03c7b9934b214ce6d09fcfaa8547" + +# aws opensearch associate-package \ +# --region us-east-1 \ +# --domain-name osi-ubi-domain \ +# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE +# --package-id ${PACKAGE_ID} + + +# aws opensearch list-packages-for-domain +# --domain-name osi-ubi-domain \ +# --region $REGION \ +# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ No newline at end of file diff --git a/osi/delete-indexes.sh b/osi/delete-indexes.sh new file mode 100755 index 0000000..22fcda3 --- /dev/null +++ b/osi/delete-indexes.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +# pip install awscurl + +# Get from the Terraform output or the AWS Console +OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_events,ubi_queries" \ + -X DELETE \ + --region us-east-1 \ + --service es \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE diff --git a/osi/get-indexed-events.sh b/osi/get-indexed-events.sh new file mode 100755 index 0000000..b969710 --- /dev/null +++ b/osi/get-indexed-events.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +# pip install awscurl + +# Get from the Terraform output or the AWS Console +OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ + -X GET \ + --region us-east-1 \ + --service es \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_events/_search" \ + -X GET \ + --region us-east-1 \ + --service es \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE | jq diff --git a/osi/get-indexed-queries.sh b/osi/get-indexed-queries.sh new file mode 100755 index 0000000..2fd198a --- /dev/null +++ b/osi/get-indexed-queries.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +# pip install awscurl + +# Get from the Terraform output or the AWS Console +OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ + -X GET \ + --region us-east-1 \ + --service es \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \ + -X GET \ + --region us-east-1 \ + --service es \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE | jq diff --git a/osi/main.tf b/osi/main.tf new file mode 100644 index 0000000..023e981 --- /dev/null +++ b/osi/main.tf @@ -0,0 +1,195 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.36" + } + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "us-east-1" + profile = "TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE" +} + +data "aws_region" "current" {} +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id +} + +output "ingest_endpoint_url" { + value = tolist(aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls)[0] +} + +resource "aws_iam_role" "ubi" { + name = "ubiosisrole" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "osis-pipelines.amazonaws.com" + } + } + ] + }) +} + +data "aws_iam_policy_document" "access_policy" { + statement { + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["${aws_iam_role.ubi.arn}"] + } + + actions = ["es:*"] + } +} + +resource "aws_opensearch_domain" "opensearch_ubi" { + + domain_name = "osi-ubi-domain" + engine_version = "OpenSearch_2.17" + + cluster_config { + instance_type = "t3.small.search" + } + + encrypt_at_rest { + enabled = true + } + + domain_endpoint_options { + enforce_https = true + tls_security_policy = "Policy-Min-TLS-1-2-2019-07" + } + + node_to_node_encryption { + enabled = true + } + + ebs_options { + ebs_enabled = true + volume_size = 10 + } + + access_policies = data.aws_iam_policy_document.access_policy.json +} + +resource "aws_iam_policy" "ubi" { + name = "osis_role_policy" + description = "Policy for OSIS pipeline role" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = ["es:DescribeDomain"] + Effect = "Allow" + Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/*" + }, + { + Action = ["es:ESHttp*"] + Effect = "Allow" + Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/osi-ubi-domain/*" + }, + { + Action = ["s3:PutObject"] + Effect = "Allow" + Resource = "arn:aws:s3:::${aws_s3_bucket.ubi_queries_events_bucket.id}/*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "ubi" { + role = aws_iam_role.ubi.name + policy_arn = aws_iam_policy.ubi.arn +} + +resource "aws_cloudwatch_log_group" "ubi" { + name = "/aws/vendedlogs/OpenSearchIngestion/ubi-pipeline" + retention_in_days = 14 + tags = { + Name = "UBI OSIS Pipeline Log Group" + } +} + +resource "aws_s3_bucket" "ubi_queries_events_bucket" { + bucket = "ubi-queries-events-sink" +} + +resource "aws_osis_pipeline" "ubi_events_pipeline" { + pipeline_name = "ubi-pipeline" + pipeline_configuration_body = <<-EOT + version: "2" + ubi-pipeline: + source: + http: + path: "/ubi" + processor: + - date: + from_time_received: true + destination: "@timestamp" + route: + - ubi-events: '/type == "event"' + - ubi-queries: '/type == "query"' + sink: + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_events" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-events] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_events/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-events] + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_queries" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-queries] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_queries/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-queries] + EOT + max_units = 1 + min_units = 1 + log_publishing_options { + is_logging_enabled = true + cloudwatch_log_destination { + log_group = aws_cloudwatch_log_group.ubi.name + } + } + tags = { + Name = "UBI OpenSearch Ingestion Pipeline for UBI" + } +} \ No newline at end of file diff --git a/osi/send-event.sh b/osi/send-event.sh new file mode 100755 index 0000000..b7a1823 --- /dev/null +++ b/osi/send-event.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e + +# pip install awscurl + +# Get from the Terraform output or the AWS Console +OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com" + +awscurl \ + --service osis \ + --region us-east-1 \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + -X POST \ + -H "Content-Type: application/json" \ + -d '[{"type": "event", "action_name": "click", "query_id": "99999999-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \ + https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi diff --git a/osi/send-query.sh b/osi/send-query.sh new file mode 100755 index 0000000..9b41884 --- /dev/null +++ b/osi/send-query.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e + +# pip install awscurl + +# Get from the Terraform output or the AWS Console +OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com" + +awscurl \ + --service osis \ + --region us-east-1 \ + --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + -X POST \ + -H "Content-Type: application/json" \ + -d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \ + https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi From a24144a58551423cb5f4c21f2619152ca2e25539 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 23 Dec 2024 09:19:26 -0500 Subject: [PATCH 2/4] Removing hardcoded values. Signed-off-by: jzonthemtn --- osi/create-plugin-package.sh | 23 ++++++++++++++--------- osi/delete-indexes.sh | 9 ++++----- osi/get-indexed-events.sh | 13 ++++++------- osi/get-indexed-queries.sh | 13 ++++++------- osi/main.tf | 12 ++++++++++-- osi/requirements.txt | 1 + osi/send-event.sh | 9 ++++----- osi/send-query.sh | 9 ++++----- osi/stack.properties | 5 +++++ 9 files changed, 54 insertions(+), 40 deletions(-) create mode 100644 osi/requirements.txt create mode 100644 osi/stack.properties diff --git a/osi/create-plugin-package.sh b/osi/create-plugin-package.sh index 6584a14..f3b70da 100755 --- a/osi/create-plugin-package.sh +++ b/osi/create-plugin-package.sh @@ -1,5 +1,10 @@ #!/bin/bash -e +. stack.properties + +# This script is just here for an example. +# AWS OpenSearch does not (yet) support ActionPlugins. + # https://docs.aws.amazon.com/opensearch-service/latest/developerguide/custom-plugins.html # Note that custom plugins cannot implement ActionPlugin. @@ -8,20 +13,20 @@ FILE_NAME="search-quality-evaluation-plugin-0.0.1.zip" wget -O ${FILE_NAME} https://github.com/o19s/opensearch-search-quality-evaluation/releases/download/0.0.1/${FILE_NAME} aws s3 cp ${FILE_NAME} s3://ubi-queries-events-sink/${FILE_NAME} \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ - --region us-east-1 + --profile ${AWS_PROFILE} \ + --region ${AWS_REGION} aws opensearch create-package \ - --region us-east-1 \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + --region ${AWS_REGION} \ + --profile ${AWS_PROFILE} \ --package-name search-eval-framework \ --package-type ZIP-PLUGIN \ --package-source S3BucketName=ubi-queries-events-sink,S3Key=search-quality-evaluation-plugin-0.0.1.zip \ --engine-version OpenSearch_2.17 # aws opensearch describe-packages \ -# --region us-east-1 \ -# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ +# --region ${AWS_REGION} \ +# --profile ${AWS_PROFILE} \ # --filters '[{"Name": "PackageType","Value": ["ZIP-PLUGIN"]}, {"Name": "PackageName","Value": ["search-eval-framework"]}]' # when done, grab the package id and put into command: @@ -29,13 +34,13 @@ aws opensearch create-package \ # PACKAGE_ID="pkg-b618759e2c2d03c7b9934b214ce6d09fcfaa8547" # aws opensearch associate-package \ -# --region us-east-1 \ +# --region ${AWS_REGION} \ # --domain-name osi-ubi-domain \ -# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE +# --profile ${AWS_PROFILE} # --package-id ${PACKAGE_ID} # aws opensearch list-packages-for-domain # --domain-name osi-ubi-domain \ # --region $REGION \ -# --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ No newline at end of file +# --profile ${AWS_PROFILE} \ No newline at end of file diff --git a/osi/delete-indexes.sh b/osi/delete-indexes.sh index 22fcda3..a8bf344 100755 --- a/osi/delete-indexes.sh +++ b/osi/delete-indexes.sh @@ -1,13 +1,12 @@ #!/bin/bash -e -# pip install awscurl +. stck.properties -# Get from the Terraform output or the AWS Console -OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` awscurl \ "https://${OPENSEARCH_ENDPOINT}/ubi_events,ubi_queries" \ -X DELETE \ - --region us-east-1 \ + --region ${AWS_REGION} \ --service es \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE + --profile ${AWS_PROFILE} diff --git a/osi/get-indexed-events.sh b/osi/get-indexed-events.sh index b969710..629e2b7 100755 --- a/osi/get-indexed-events.sh +++ b/osi/get-indexed-events.sh @@ -1,20 +1,19 @@ #!/bin/bash -e -# pip install awscurl +. stack.properties -# Get from the Terraform output or the AWS Console -OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` awscurl \ "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ -X GET \ - --region us-east-1 \ + --region ${AWS_REGION} \ --service es \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE + --profile ${AWS_PROFILE} awscurl \ "https://${OPENSEARCH_ENDPOINT}/ubi_events/_search" \ -X GET \ - --region us-east-1 \ + --region ${AWS_REGION} \ --service es \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE | jq + --profile ${AWS_PROFILE} | jq diff --git a/osi/get-indexed-queries.sh b/osi/get-indexed-queries.sh index 2fd198a..a8d3be0 100755 --- a/osi/get-indexed-queries.sh +++ b/osi/get-indexed-queries.sh @@ -1,20 +1,19 @@ #!/bin/bash -e -# pip install awscurl +. stack.properties -# Get from the Terraform output or the AWS Console -OPENSEARCH_ENDPOINT="search-osi-ubi-domain-pjju5yl7neorgz4jcsqhq5o7fq.us-east-1.es.amazonaws.com" +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` awscurl \ "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ -X GET \ - --region us-east-1 \ + --region ${AWS_REGION} \ --service es \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE + --profile ${AWS_PROFILE} awscurl \ "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \ -X GET \ - --region us-east-1 \ + --region ${AWS_REGION} \ --service es \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE | jq + --profile ${AWS_PROFILE} | jq diff --git a/osi/main.tf b/osi/main.tf index 023e981..14db4ac 100644 --- a/osi/main.tf +++ b/osi/main.tf @@ -11,7 +11,7 @@ terraform { provider "aws" { region = "us-east-1" - profile = "TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE" + profile = "mtnfog" } data "aws_region" "current" {} @@ -81,7 +81,7 @@ resource "aws_opensearch_domain" "opensearch_ubi" { volume_size = 10 } - access_policies = data.aws_iam_policy_document.access_policy.json + #access_policies = data.aws_iam_policy_document.access_policy.json } resource "aws_iam_policy" "ubi" { @@ -192,4 +192,12 @@ resource "aws_osis_pipeline" "ubi_events_pipeline" { tags = { Name = "UBI OpenSearch Ingestion Pipeline for UBI" } +} + +output "opensearch_domain_endpoint" { + value = aws_opensearch_domain.opensearch_ubi.endpoint +} + +output "opensearch_ingest_pipeline_endpoint" { + value = aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls } \ No newline at end of file diff --git a/osi/requirements.txt b/osi/requirements.txt new file mode 100644 index 0000000..7929f08 --- /dev/null +++ b/osi/requirements.txt @@ -0,0 +1 @@ +awscurl==0.36 \ No newline at end of file diff --git a/osi/send-event.sh b/osi/send-event.sh index b7a1823..fa001eb 100755 --- a/osi/send-event.sh +++ b/osi/send-event.sh @@ -1,14 +1,13 @@ #!/bin/bash -e -# pip install awscurl +. stack.properties -# Get from the Terraform output or the AWS Console -OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com" +OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]` awscurl \ --service osis \ - --region us-east-1 \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + --region ${AWS_REGION} \ + --profile ${AWS_PROFILE} \ -X POST \ -H "Content-Type: application/json" \ -d '[{"type": "event", "action_name": "click", "query_id": "99999999-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \ diff --git a/osi/send-query.sh b/osi/send-query.sh index 9b41884..6a91434 100755 --- a/osi/send-query.sh +++ b/osi/send-query.sh @@ -1,14 +1,13 @@ #!/bin/bash -e -# pip install awscurl +. stack.properties -# Get from the Terraform output or the AWS Console -OSIS_PIPELINE_ENDPOINT_URL="ubi-pipeline-xjmot6taz7mmcv76a36mlscgg4.us-east-1.osis.amazonaws.com" +OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]` awscurl \ --service osis \ - --region us-east-1 \ - --profile TODO_PUT_YOUR_AWS_PROFILE_NAME_HERE \ + --region ${AWS_REGION} \ + --profile ${AWS_PROFILE} \ -X POST \ -H "Content-Type: application/json" \ -d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \ diff --git a/osi/stack.properties b/osi/stack.properties new file mode 100644 index 0000000..a797cad --- /dev/null +++ b/osi/stack.properties @@ -0,0 +1,5 @@ +# The name of the AWS profile. +AWS_PROFILE=mtnfog + +# The AWS region. +AWS_REGION=us-east-1 From 49ec839af6472c7793f3399d87a6e7c203b110af Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 23 Dec 2024 12:59:34 -0500 Subject: [PATCH 3/4] Fixing file name. Signed-off-by: jzonthemtn --- osi/delete-indexes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osi/delete-indexes.sh b/osi/delete-indexes.sh index a8bf344..573eed8 100755 --- a/osi/delete-indexes.sh +++ b/osi/delete-indexes.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -. stck.properties +. stack.properties OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` From 4889864250274ef584944e1252ca20068531fa8b Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sun, 19 Jan 2025 15:02:27 -0500 Subject: [PATCH 4/4] Adding osi blueprint. --- osi/blueprint.yaml | 51 ++++++++++++++++++++++++++++++++++ osi/create-plugin-package.sh | 46 ------------------------------ osi/main.tf | 54 +----------------------------------- 3 files changed, 52 insertions(+), 99 deletions(-) create mode 100644 osi/blueprint.yaml delete mode 100755 osi/create-plugin-package.sh diff --git a/osi/blueprint.yaml b/osi/blueprint.yaml new file mode 100644 index 0000000..5e82dfd --- /dev/null +++ b/osi/blueprint.yaml @@ -0,0 +1,51 @@ +version: "2" +ubi-pipeline: + source: + http: + path: "/ubi" + processor: + - date: + from_time_received: true + destination: "@timestamp" + route: + - ubi-events: '/type == "event"' + - ubi-queries: '/type == "query"' + sink: + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_events" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-events] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_events/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-events] + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_queries" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-queries] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_queries/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-queries] diff --git a/osi/create-plugin-package.sh b/osi/create-plugin-package.sh deleted file mode 100755 index f3b70da..0000000 --- a/osi/create-plugin-package.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -e - -. stack.properties - -# This script is just here for an example. -# AWS OpenSearch does not (yet) support ActionPlugins. - -# https://docs.aws.amazon.com/opensearch-service/latest/developerguide/custom-plugins.html -# Note that custom plugins cannot implement ActionPlugin. - -FILE_NAME="search-quality-evaluation-plugin-0.0.1.zip" - -wget -O ${FILE_NAME} https://github.com/o19s/opensearch-search-quality-evaluation/releases/download/0.0.1/${FILE_NAME} - -aws s3 cp ${FILE_NAME} s3://ubi-queries-events-sink/${FILE_NAME} \ - --profile ${AWS_PROFILE} \ - --region ${AWS_REGION} - -aws opensearch create-package \ - --region ${AWS_REGION} \ - --profile ${AWS_PROFILE} \ - --package-name search-eval-framework \ - --package-type ZIP-PLUGIN \ - --package-source S3BucketName=ubi-queries-events-sink,S3Key=search-quality-evaluation-plugin-0.0.1.zip \ - --engine-version OpenSearch_2.17 - -# aws opensearch describe-packages \ -# --region ${AWS_REGION} \ -# --profile ${AWS_PROFILE} \ -# --filters '[{"Name": "PackageType","Value": ["ZIP-PLUGIN"]}, {"Name": "PackageName","Value": ["search-eval-framework"]}]' - -# when done, grab the package id and put into command: - -# PACKAGE_ID="pkg-b618759e2c2d03c7b9934b214ce6d09fcfaa8547" - -# aws opensearch associate-package \ -# --region ${AWS_REGION} \ -# --domain-name osi-ubi-domain \ -# --profile ${AWS_PROFILE} -# --package-id ${PACKAGE_ID} - - -# aws opensearch list-packages-for-domain -# --domain-name osi-ubi-domain \ -# --region $REGION \ -# --profile ${AWS_PROFILE} \ No newline at end of file diff --git a/osi/main.tf b/osi/main.tf index 14db4ac..9955587 100644 --- a/osi/main.tf +++ b/osi/main.tf @@ -128,59 +128,7 @@ resource "aws_s3_bucket" "ubi_queries_events_bucket" { resource "aws_osis_pipeline" "ubi_events_pipeline" { pipeline_name = "ubi-pipeline" - pipeline_configuration_body = <<-EOT - version: "2" - ubi-pipeline: - source: - http: - path: "/ubi" - processor: - - date: - from_time_received: true - destination: "@timestamp" - route: - - ubi-events: '/type == "event"' - - ubi-queries: '/type == "query"' - sink: - - opensearch: - hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] - index: "ubi_events" - aws: - sts_role_arn: "${aws_iam_role.ubi.arn}" - region: "${data.aws_region.current.name}" - routes: [ubi-events] - - s3: - aws: - sts_role_arn: "${aws_iam_role.ubi.arn}" - region: "${data.aws_region.current.name}" - bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" - object_key: - path_prefix: ubi_events/ - threshold: - event_collect_timeout: "60s" - codec: - ndjson: - routes: [ubi-events] - - opensearch: - hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] - index: "ubi_queries" - aws: - sts_role_arn: "${aws_iam_role.ubi.arn}" - region: "${data.aws_region.current.name}" - routes: [ubi-queries] - - s3: - aws: - sts_role_arn: "${aws_iam_role.ubi.arn}" - region: "${data.aws_region.current.name}" - bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" - object_key: - path_prefix: ubi_queries/ - threshold: - event_collect_timeout: "60s" - codec: - ndjson: - routes: [ubi-queries] - EOT + pipeline_configuration_body = file("blueprint.yaml") max_units = 1 min_units = 1 log_publishing_options {