Feature/primary mapped (#92)

* fix to create ecs aws_iam_role_policy when deploying on ecs * configure primary metastore mapped databases * fix * fix * update changelog Co-authored-by: Raj Poluri <[email protected]>
ExpediaGroup · Nov 1, 2021 · fa1d55c · fa1d55c
1 parent b810982
commit fa1d55c
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,19 +3,23 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [3.3.9] - 2021-010-27
+## [3.3.10] - 2021-11-01
+### Changed
+- Variable to configure pimary metastore mapped databases.
+
+## [3.3.9] - 2021-10-27
 ### Fixed
 - Fix to create ECS aws_iam_role_policy only when deploying on ECS.
 
-## [3.3.8] - 2021-010-18
+## [3.3.8] - 2021-10-18
 ### Fixed
 - Fix kubernetes serviceaccount reference.
 
-## [3.3.7] - 2021-010-18
+## [3.3.7] - 2021-10-18
 ### Fixed
 - Disable kubernetes serviceaccount creation when running on ecs.
 
-## [3.3.6] - 2021-010-18
+## [3.3.6] - 2021-10-18
 ### Added
 - Support Waggledance `latency` parameter
 

diff --git a/README.md b/README.md
@@ -29,6 +29,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 | primary_metastore_host | Primary Hive Metastore hostname configured in Waggle Dance. | string | `localhost` | no |
 | primary_metastore_port | Primary Hive Metastore port | string | `9083` | no |
 | primary_metastore_whitelist | List of Hive databases to whitelist on primary Metastore. | list | `<list>` | no |
+| primary_metastore_mapped_databases | List of Hive databases mapped from primary Metastore. | list | `<list>` | no |
 | remote_metastores | List of VPC endpoint services to federate Metastores in other accounts. See section [`remote_metastores`](#remote_metastores) for more info.| list | `<list>` | no |
 | remote_region_metastores | List of VPC endpoint services to federate Metastores in other region,other accounts. The actual data from tables in these metastores can be accessed using Alluxio caching instead of reading the data from S3 directly. See section [`remote_region_metastores`](#remote_region_metastores) for more info.| list | `<list>` | no |
 | secondary_vpcs | List of VPCs to associate with Service Discovery namespace. | list | `<list>` | no |

diff --git a/templates.tf b/templates.tf
@@ -45,6 +45,14 @@ data "template_file" "primary_metastore_whitelist" {
 EOF
 }
 
+data "template_file" "primary_metastore_mapped_databases" {
+  count = length(var.primary_metastore_mapped_databases)
+
+  template = <<EOF
+  - ${var.primary_metastore_mapped_databases[count.index]}
+EOF
+}
+
 data "template_file" "local_metastores_yaml" {
   count    = length(var.local_metastores)
   template = file("${path.module}/templates/waggle-dance-federation-local.yml.tmpl")
@@ -120,14 +128,15 @@ data "template_file" "federation_yaml" {
   template = file("${path.module}/templates/waggle-dance-federation.yml.tmpl")
 
   vars = {
-    primary_metastore_host      = var.primary_metastore_host
-    primary_metastore_port      = var.primary_metastore_port
-    primary_metastore_latency   = var.default_latency
-    primary_metastore_whitelist = join("", data.template_file.primary_metastore_whitelist.*.rendered)
-    local_metastores            = join("", data.template_file.local_metastores_yaml.*.rendered)
-    remote_metastores           = join("", data.template_file.remote_metastores_yaml.*.rendered)
-    remote_region_metastores    = join("", data.template_file.remote_region_metastores_yaml.*.rendered)
-    ssh_metastores              = join("", data.template_file.ssh_metastores_yaml.*.rendered)
+    primary_metastore_host             = var.primary_metastore_host
+    primary_metastore_port             = var.primary_metastore_port
+    primary_metastore_latency          = var.default_latency
+    primary_metastore_whitelist        = join("", data.template_file.primary_metastore_whitelist.*.rendered)
+    primary_metastore_mapped_databases = join("", data.template_file.primary_metastore_mapped_databases.*.rendered)
+    local_metastores                   = join("", data.template_file.local_metastores_yaml.*.rendered)
+    remote_metastores                  = join("", data.template_file.remote_metastores_yaml.*.rendered)
+    remote_region_metastores           = join("", data.template_file.remote_region_metastores_yaml.*.rendered)
+    ssh_metastores                     = join("", data.template_file.ssh_metastores_yaml.*.rendered)
   }
 }
 

diff --git a/templates/waggle-dance-federation.yml.tmpl b/templates/waggle-dance-federation.yml.tmpl
@@ -4,6 +4,10 @@ primary-meta-store:
   name: primary
   remote-meta-store-uris: thrift://${primary_metastore_host}:${primary_metastore_port}
   latency: ${primary_metastore_latency}
+%{if primary_metastore_mapped_databases != "" ~}
+  mapped-databases:
+${primary_metastore_mapped_databases}
+%{~endif~}
   writable-database-white-list:
 ${primary_metastore_whitelist}
 federated-meta-stores:

diff --git a/variables.tf b/variables.tf
@@ -163,6 +163,12 @@ variable "primary_metastore_whitelist" {
   default     = ["default"]
 }
 
+variable "primary_metastore_mapped_databases" {
+  description = "List of Hive databases mapped from primary Metastore."
+  type        = list(string)
+  default     = []
+}
+
 #list of maps, example: [ {host="metastore1", port="9083", prefix="pre1", writable-whitelist="db1,test" }, {host="metastore2", port="9083", prefix="pre2", mapped-databases="dm,test" } ]
 variable "local_metastores" {
   description = "List of federated Metastore endpoints directly accessible on the local network."
@@ -243,4 +249,4 @@ variable "default_latency" {
   type        = number
   default     = 0
   description = "HMS latency (in ms.) that Waggledance will tolerate.  See \"latency\" parameter in https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md."
-}
+}