From c07d42e110b4012d2410acfbe6ceb6f4ac5ec2a4 Mon Sep 17 00:00:00 2001 From: Matthew Skinner Date: Wed, 16 Oct 2024 16:15:17 -0500 Subject: [PATCH] feature(TPG>=5.39)!: add resource_tags (#354) Co-authored-by: Anuj Rohilla Co-authored-by: Anuj Rohilla <163002505+rohilla-anuj@users.noreply.github.com> --- README.md | 2 + main.tf | 1 + metadata.display.yaml | 3 + metadata.yaml | 189 ++++++++++++------------ modules/authorization/metadata.yaml | 45 +++--- modules/data_warehouse/metadata.yaml | 45 +++--- modules/scheduled_queries/metadata.yaml | 3 + modules/udf/metadata.yaml | 11 +- variables.tf | 6 + versions.tf | 2 +- 10 files changed, 169 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 0f1393d4..7892fe08 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ module "bigquery" { project_id = "" location = "US" default_table_expiration_ms = 3600000 + resource_tags = {"/":""} tables = [ { @@ -199,6 +200,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | 
list(object({
    view_id             = string,
    description         = optional(string),
    query               = string,
    enable_refresh      = bool,
    refresh_interval_ms = string,
    clustering          = list(string),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
      require_partition_filter = bool,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    max_staleness   = optional(string),
    labels          = map(string),
  }))
| `[]` | no | | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | +| resource\_tags | A map of resource tags to add to the dataset | `map(string)` | `{}` | no | | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | 
list(object({
    routine_id      = string,
    routine_type    = string,
    language        = string,
    definition_body = string,
    return_type     = string,
    description     = string,
    arguments = list(object({
      name          = string,
      data_type     = string,
      argument_kind = string,
      mode          = string,
    })),
  }))
| `[]` | no | | tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | 
list(object({
    table_id                 = string,
    description              = optional(string),
    table_name               = optional(string),
    schema                   = string,
    clustering               = list(string),
    require_partition_filter = optional(bool),
    time_partitioning = object({
      expiration_ms = string,
      field         = string,
      type          = string,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time     = string,
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | views | A list of objects which include view\_id and view query | 
list(object({
    view_id        = string,
    description    = optional(string),
    query          = string,
    use_legacy_sql = bool,
    labels         = map(string),
  }))
| `[]` | no | diff --git a/main.tf b/main.tf index aa7a6297..aa692341 100644 --- a/main.tf +++ b/main.tf @@ -38,6 +38,7 @@ resource "google_bigquery_dataset" "main" { max_time_travel_hours = var.max_time_travel_hours project = var.project_id labels = var.dataset_labels + resource_tags = var.resource_tags dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [var.encryption_key] diff --git a/metadata.display.yaml b/metadata.display.yaml index 81c5ec4a..3efb9c25 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -69,6 +69,9 @@ spec: project_id: name: project_id title: Project Id + resource_tags: + name: resource_tags + title: Resource Tags routines: name: routines title: Routines diff --git a/metadata.yaml b/metadata.yaml index ab23c9aa..1c79795d 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -52,26 +52,20 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: access - description: An array of objects that define dataset access for one or more entities. - varType: any - defaultValue: - - role: roles/bigquery.dataOwner - special_group: projectOwners - name: dataset_id description: Unique ID for the dataset being provisioned. varType: string required: true - - name: dataset_labels - description: Key value pairs in a map for dataset labels - varType: map(string) - defaultValue: {} - name: dataset_name description: Friendly name for the dataset being provisioned. varType: string - - name: default_table_expiration_ms - description: TTL of tables using the dataset in MS - varType: number + - name: description + description: Dataset description. + varType: string + - name: location + description: The regional location for the dataset only US and EU are allowed in module + varType: string + defaultValue: US - name: delete_contents_on_destroy description: (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. varType: bool @@ -79,51 +73,72 @@ spec: description: Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion_protection configuration. varType: bool defaultValue: false - - name: description - description: Dataset description. + - name: default_table_expiration_ms + description: TTL of tables using the dataset in MS + varType: number + - name: max_time_travel_hours + description: Defines the time travel window in hours + varType: number + - name: project_id + description: Project where the dataset and table are created varType: string + required: true - name: encryption_key description: Default encryption key to apply to the dataset. Defaults to null (Google-managed). varType: string - - name: external_tables - description: A list of objects which include table_id, expiration_time, external_data_configuration, and labels. + - name: dataset_labels + description: Key value pairs in a map for dataset labels + varType: map(string) + defaultValue: {} + - name: resource_tags + description: A map of resource tags to add to the dataset + varType: map(string) + defaultValue: {} + - name: access + description: An array of objects that define dataset access for one or more entities. + varType: any + defaultValue: + - role: roles/bigquery.dataOwner + special_group: projectOwners + - name: tables + description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. varType: |- list(object({ - table_id = string, - description = optional(string), - autodetect = bool, - compression = string, - ignore_unknown_values = bool, - max_bad_records = number, - schema = string, - source_format = string, - source_uris = list(string), - csv_options = object({ - quote = string, - allow_jagged_rows = bool, - allow_quoted_newlines = bool, - encoding = string, - field_delimiter = string, - skip_leading_rows = number, - }), - google_sheets_options = object({ - range = string, - skip_leading_rows = number, + table_id = string, + description = optional(string), + table_name = optional(string), + schema = string, + clustering = list(string), + require_partition_filter = optional(bool), + time_partitioning = object({ + expiration_ms = string, + field = string, + type = string, }), - hive_partitioning_options = object({ - mode = string, - source_uri_prefix = string, + range_partitioning = object({ + field = string, + range = object({ + start = string, + end = string, + interval = string, + }), }), expiration_time = string, - max_staleness = optional(string), deletion_protection = optional(bool), labels = map(string), })) defaultValue: [] - - name: location - description: The regional location for the dataset only US and EU are allowed in module - varType: string - defaultValue: US + - name: views + description: A list of objects which include view_id and view query + varType: |- + list(object({ + view_id = string, + description = optional(string), + query = string, + use_legacy_sql = bool, + labels = map(string), + })) + defaultValue: [] - name: materialized_views description: A list of objects which includes view_id, view_query, clustering, time_partitioning, range_partitioning, expiration_time and labels varType: |- @@ -153,13 +168,41 @@ spec: labels = map(string), })) defaultValue: [] - - name: max_time_travel_hours - description: Defines the time travel window in hours - varType: number - - name: project_id - description: Project where the dataset and table are created - varType: string - required: true + - name: external_tables + description: A list of objects which include table_id, expiration_time, external_data_configuration, and labels. + varType: |- + list(object({ + table_id = string, + description = optional(string), + autodetect = bool, + compression = string, + ignore_unknown_values = bool, + max_bad_records = number, + schema = string, + source_format = string, + source_uris = list(string), + csv_options = object({ + quote = string, + allow_jagged_rows = bool, + allow_quoted_newlines = bool, + encoding = string, + field_delimiter = string, + skip_leading_rows = number, + }), + google_sheets_options = object({ + range = string, + skip_leading_rows = number, + }), + hive_partitioning_options = object({ + mode = string, + source_uri_prefix = string, + }), + expiration_time = string, + max_staleness = optional(string), + deletion_protection = optional(bool), + labels = map(string), + })) + defaultValue: [] - name: routines description: A list of objects which include routine_id, routine_type, routine_language, definition_body, return_type, routine_description and arguments. varType: |- @@ -178,45 +221,6 @@ spec: })), })) defaultValue: [] - - name: tables - description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. - varType: |- - list(object({ - table_id = string, - description = optional(string), - table_name = optional(string), - schema = string, - clustering = list(string), - require_partition_filter = optional(bool), - time_partitioning = object({ - expiration_ms = string, - field = string, - type = string, - }), - range_partitioning = object({ - field = string, - range = object({ - start = string, - end = string, - interval = string, - }), - }), - expiration_time = string, - deletion_protection = optional(bool), - labels = map(string), - })) - defaultValue: [] - - name: views - description: A list of objects which include view_id and view query - varType: |- - list(object({ - view_id = string, - description = optional(string), - query = string, - use_legacy_sql = bool, - labels = map(string), - })) - defaultValue: [] outputs: - name: bigquery_dataset description: Bigquery dataset resource. @@ -257,3 +261,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/authorization/metadata.yaml b/modules/authorization/metadata.yaml index b528faa6..d18c7f1f 100644 --- a/modules/authorization/metadata.yaml +++ b/modules/authorization/metadata.yaml @@ -44,6 +44,27 @@ spec: location: examples/scheduled_queries interfaces: variables: + - name: dataset_id + description: Unique ID for the dataset being provisioned. + varType: string + required: true + - name: project_id + description: Project where the dataset and table are created + varType: string + required: true + - name: roles + description: An array of objects that define dataset access for one or more entities. + varType: any + defaultValue: [] + - name: authorized_views + description: An array of views to give authorize for the dataset + varType: |- + list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + defaultValue: [] - name: authorized_datasets description: An array of datasets to be authorized on the dataset varType: |- @@ -61,27 +82,6 @@ spec: routine_id = string })) defaultValue: [] - - name: authorized_views - description: An array of views to give authorize for the dataset - varType: |- - list(object({ - dataset_id = string, - project_id = string, - table_id = string # this is the view id, but we keep table_id to stay consistent as the resource - })) - defaultValue: [] - - name: dataset_id - description: Unique ID for the dataset being provisioned. - varType: string - required: true - - name: project_id - description: Project where the dataset and table are created - varType: string - required: true - - name: roles - description: An array of objects that define dataset access for one or more entities. - varType: any - defaultValue: [] outputs: - name: authorized_dataset description: Authorized datasets for the BQ dataset @@ -104,3 +104,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/data_warehouse/metadata.yaml b/modules/data_warehouse/metadata.yaml index b8810886..eb152de3 100644 --- a/modules/data_warehouse/metadata.yaml +++ b/modules/data_warehouse/metadata.yaml @@ -77,17 +77,26 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: create_ignore_service_accounts - description: Whether or not to ignore creation of a service account if an account of the same name already exists + - name: project_id + description: Google Cloud Project ID varType: string - defaultValue: true + required: true + - name: region + description: Google Cloud Region + varType: string + required: true - name: dataform_region description: Region that is used to deploy Dataform resources. This does not limit where resources can be run or what region data must be located in. varType: string - - name: deletion_protection - description: Whether or not to protect GCS resources from deletion when solution is modified or changed. + - name: text_generation_model_name + description: Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation varType: string - defaultValue: false + defaultValue: text_generate_model + - name: labels + description: A map of labels to apply to contained resources. + varType: map(string) + defaultValue: + data-warehouse: true - name: enable_apis description: Whether or not to enable underlying apis in this solution. varType: string @@ -96,23 +105,14 @@ spec: description: Whether or not to protect BigQuery resources from deletion when solution is modified or changed. varType: string defaultValue: true - - name: labels - description: A map of labels to apply to contained resources. - varType: map(string) - defaultValue: - data-warehouse: true - - name: project_id - description: Google Cloud Project ID - varType: string - required: true - - name: region - description: Google Cloud Region + - name: deletion_protection + description: Whether or not to protect GCS resources from deletion when solution is modified or changed. varType: string - required: true - - name: text_generation_model_name - description: Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation + defaultValue: false + - name: create_ignore_service_accounts + description: Whether or not to ignore creation of a service account if an account of the same name already exists varType: string - defaultValue: text_generate_model + defaultValue: true outputs: - name: bigquery_editor_url description: The URL to launch the BigQuery editor with the sample query procedure opened @@ -139,3 +139,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/scheduled_queries/metadata.yaml b/modules/scheduled_queries/metadata.yaml index d81af5f8..230e3223 100644 --- a/modules/scheduled_queries/metadata.yaml +++ b/modules/scheduled_queries/metadata.yaml @@ -70,3 +70,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/modules/udf/metadata.yaml b/modules/udf/metadata.yaml index be290015..013362cf 100644 --- a/modules/udf/metadata.yaml +++ b/modules/udf/metadata.yaml @@ -44,10 +44,6 @@ spec: location: examples/scheduled_queries interfaces: variables: - - name: add_udfs - description: Whether or not this module should be enabled. - varType: string - defaultValue: false - name: dataset_id description: Dataset id varType: string @@ -56,6 +52,10 @@ spec: description: Project ID that contains the dataset varType: string required: true + - name: add_udfs + description: Whether or not this module should be enabled. + varType: string + defaultValue: false outputs: - name: added_udfs description: List of UDFs utility functions added. @@ -74,3 +74,6 @@ spec: - bigqueryconnection.googleapis.com - serviceusage.googleapis.com - iam.googleapis.com + providerVersions: + - source: hashicorp/google + version: ">= 5.39, < 7" diff --git a/variables.tf b/variables.tf index 68a0667d..02dcc6f6 100644 --- a/variables.tf +++ b/variables.tf @@ -78,6 +78,12 @@ variable "dataset_labels" { default = {} } +variable "resource_tags" { + description = "A map of resource tags to add to the dataset" + type = map(string) + default = {} +} + # Format: list(objects) # domain: A domain to grant access to. # group_by_email: An email address of a Google Group to grant access to. diff --git a/versions.tf b/versions.tf index bb8bc259..21d80fc4 100644 --- a/versions.tf +++ b/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = ">= 5.3, < 7" + version = ">= 5.39, < 7" } }