From 20bb01a70d0ec7167c1d5a2ab770787913bf7696 Mon Sep 17 00:00:00 2001 From: SamuZad Date: Wed, 3 Jul 2024 10:03:53 +0100 Subject: [PATCH 1/4] add storage_billing_model --- README.md | 1 + main.tf | 1 + metadata.display.yaml | 3 +++ metadata.yaml | 3 +++ variables.tf | 11 +++++++++++ 5 files changed, 19 insertions(+) diff --git a/README.md b/README.md index 08a5318a..60a282e2 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | 
list(object({
    routine_id      = string,
    routine_type    = string,
    language        = string,
    definition_body = string,
    return_type     = string,
    description     = string,
    arguments = list(object({
      name          = string,
      data_type     = string,
      argument_kind = string,
      mode          = string,
    })),
  }))
| `[]` | no | +| storage\_billing\_model | Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead | `string` | `null` | no | | tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | 
list(object({
    table_id    = string,
    description = optional(string),
    table_name  = optional(string),
    schema      = string,
    clustering  = list(string),
    time_partitioning = object({
      expiration_ms            = string,
      field                    = string,
      type                     = string,
      require_partition_filter = bool,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time     = string,
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | views | A list of objects which include view\_id and view query | 
list(object({
    view_id        = string,
    description    = optional(string),
    query          = string,
    use_legacy_sql = bool,
    labels         = map(string),
  }))
| `[]` | no | diff --git a/main.tf b/main.tf index 7c67f456..7fbbcd30 100644 --- a/main.tf +++ b/main.tf @@ -36,6 +36,7 @@ resource "google_bigquery_dataset" "main" { delete_contents_on_destroy = var.delete_contents_on_destroy default_table_expiration_ms = var.default_table_expiration_ms max_time_travel_hours = var.max_time_travel_hours + storage_billing_model = var.storage_billing_model project = var.project_id labels = var.dataset_labels diff --git a/metadata.display.yaml b/metadata.display.yaml index 81c5ec4a..a0630782 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -72,6 +72,9 @@ spec: routines: name: routines title: Routines + storage_billing_model: + name: storage_billing_model + title: Storage Billing Model tables: name: tables title: Tables diff --git a/metadata.yaml b/metadata.yaml index 7086be46..94a4fd45 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -178,6 +178,9 @@ spec: })), })) defaultValue: [] + - name: storage_billing_model + description: Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead + varType: string - name: tables description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. varType: |- diff --git a/variables.tf b/variables.tf index 3c878547..514ef2f0 100644 --- a/variables.tf +++ b/variables.tf @@ -61,6 +61,17 @@ variable "max_time_travel_hours" { default = null } +variable "storage_billing_model" { + description = "Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified." + type = string + default = null + + validation { + condition = var.storage_billing_model == null || var.storage_billing_model == "LOGICAL" || var.storage_billing_model == "PHYSICAL" + error_message = "storage_billing_model must be null, \"LOGICAL\" or \"PHYSICAL\"." + } +} + variable "project_id" { description = "Project where the dataset and table are created" type = string From f93c10bb7663e490607c1675167078bbe9ae534a Mon Sep 17 00:00:00 2001 From: Sam Poole Date: Mon, 8 Jul 2024 08:55:28 +0100 Subject: [PATCH 2/4] Add default_partition_expiration_ms --- README.md | 1 + main.tf | 21 +++++++++++---------- metadata.display.yaml | 3 +++ metadata.yaml | 3 +++ variables.tf | 6 ++++++ 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 60a282e2..dbc2e08a 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | dataset\_labels | Key value pairs in a map for dataset labels | `map(string)` | `{}` | no | | dataset\_name | Friendly name for the dataset being provisioned. | `string` | `null` | no | | default\_table\_expiration\_ms | TTL of tables using the dataset in MS | `number` | `null` | no | +| default\_partition\_expiration\_ms | The default partition expiration for all partitioned tables in the dataset, in MS | `number` | `null` | no | | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no | | deletion\_protection | Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion\_protection configuration. | `bool` | `false` | no | | description | Dataset description. | `string` | `null` | no | diff --git a/main.tf b/main.tf index 7fbbcd30..b3a90eac 100644 --- a/main.tf +++ b/main.tf @@ -29,16 +29,17 @@ locals { } resource "google_bigquery_dataset" "main" { - dataset_id = var.dataset_id - friendly_name = var.dataset_name - description = var.description - location = var.location - delete_contents_on_destroy = var.delete_contents_on_destroy - default_table_expiration_ms = var.default_table_expiration_ms - max_time_travel_hours = var.max_time_travel_hours - storage_billing_model = var.storage_billing_model - project = var.project_id - labels = var.dataset_labels + dataset_id = var.dataset_id + friendly_name = var.dataset_name + description = var.description + location = var.location + delete_contents_on_destroy = var.delete_contents_on_destroy + default_table_expiration_ms = var.default_table_expiration_ms + default_partition_expiration_ms = var.default_partition_expiration_ms + max_time_travel_hours = var.max_time_travel_hours + storage_billing_model = var.storage_billing_model + project = var.project_id + labels = var.dataset_labels dynamic "default_encryption_configuration" { for_each = var.encryption_key == null ? [] : [var.encryption_key] diff --git a/metadata.display.yaml b/metadata.display.yaml index a0630782..857df5be 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -42,6 +42,9 @@ spec: default_table_expiration_ms: name: default_table_expiration_ms title: Default Table Expiration Ms + default_partition_expiration_ms: + name: default_partition_expiration_ms + title: Default Partition Expiration Ms delete_contents_on_destroy: name: delete_contents_on_destroy title: Delete Contents On Destroy diff --git a/metadata.yaml b/metadata.yaml index 94a4fd45..f9d5cb94 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -72,6 +72,9 @@ spec: - name: default_table_expiration_ms description: TTL of tables using the dataset in MS varType: number + - name: default_partition_expiration_ms + description: The default partition expiration for all partitioned tables in the dataset, in MS + varType: number - name: delete_contents_on_destroy description: (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. varType: bool diff --git a/variables.tf b/variables.tf index 514ef2f0..f9095fa0 100644 --- a/variables.tf +++ b/variables.tf @@ -55,6 +55,12 @@ variable "default_table_expiration_ms" { default = null } +variable "default_partition_expiration_ms" { + description = "The default partition expiration for all partitioned tables in the dataset, in MS" + type = number + default = null +} + variable "max_time_travel_hours" { description = "Defines the time travel window in hours" type = number From 7fa2b72e34d1c3cce41b3970ae37090018e726a2 Mon Sep 17 00:00:00 2001 From: Awais Malik Date: Tue, 30 Jul 2024 15:51:02 -0700 Subject: [PATCH 3/4] updated docs --- README.md | 4 ++-- metadata.display.yaml | 6 +++--- metadata.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6ee475f9..ea9b1457 100644 --- a/README.md +++ b/README.md @@ -189,8 +189,8 @@ This module provisions a dataset and a list of tables with associated JSON schem | dataset\_id | Unique ID for the dataset being provisioned. | `string` | n/a | yes | | dataset\_labels | Key value pairs in a map for dataset labels | `map(string)` | `{}` | no | | dataset\_name | Friendly name for the dataset being provisioned. | `string` | `null` | no | -| default\_table\_expiration\_ms | TTL of tables using the dataset in MS | `number` | `null` | no | | default\_partition\_expiration\_ms | The default partition expiration for all partitioned tables in the dataset, in MS | `number` | `null` | no | +| default\_table\_expiration\_ms | TTL of tables using the dataset in MS | `number` | `null` | no | | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no | | deletion\_protection | Whether or not to allow deletion of tables and external tables defined by this module. Can be overriden by table-level deletion\_protection configuration. | `bool` | `false` | no | | description | Dataset description. | `string` | `null` | no | @@ -201,7 +201,7 @@ This module provisions a dataset and a list of tables with associated JSON schem | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | 
list(object({
    routine_id      = string,
    routine_type    = string,
    language        = string,
    definition_body = string,
    return_type     = string,
    description     = string,
    arguments = list(object({
      name          = string,
      data_type     = string,
      argument_kind = string,
      mode          = string,
    })),
  }))
| `[]` | no | -| storage\_billing\_model | Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead | `string` | `null` | no | +| storage\_billing\_model | Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified. | `string` | `null` | no | | tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | 
list(object({
    table_id                 = string,
    description              = optional(string),
    table_name               = optional(string),
    schema                   = string,
    clustering               = list(string),
    require_partition_filter = optional(bool),
    time_partitioning = object({
      expiration_ms = string,
      field         = string,
      type          = string,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time     = string,
    deletion_protection = optional(bool),
    labels              = map(string),
  }))
| `[]` | no | | views | A list of objects which include view\_id and view query | 
list(object({
    view_id        = string,
    description    = optional(string),
    query          = string,
    use_legacy_sql = bool,
    labels         = map(string),
  }))
| `[]` | no | diff --git a/metadata.display.yaml b/metadata.display.yaml index 857df5be..789e1846 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -39,12 +39,12 @@ spec: dataset_name: name: dataset_name title: Dataset Name - default_table_expiration_ms: - name: default_table_expiration_ms - title: Default Table Expiration Ms default_partition_expiration_ms: name: default_partition_expiration_ms title: Default Partition Expiration Ms + default_table_expiration_ms: + name: default_table_expiration_ms + title: Default Table Expiration Ms delete_contents_on_destroy: name: delete_contents_on_destroy title: Delete Contents On Destroy diff --git a/metadata.yaml b/metadata.yaml index cf6280e7..f9239e3f 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -69,12 +69,12 @@ spec: - name: dataset_name description: Friendly name for the dataset being provisioned. varType: string - - name: default_table_expiration_ms - description: TTL of tables using the dataset in MS - varType: number - name: default_partition_expiration_ms description: The default partition expiration for all partitioned tables in the dataset, in MS varType: number + - name: default_table_expiration_ms + description: TTL of tables using the dataset in MS + varType: number - name: delete_contents_on_destroy description: (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. varType: bool @@ -182,7 +182,7 @@ spec: })) defaultValue: [] - name: storage_billing_model - description: Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead + description: Specifies the storage billing model for the dataset. Set this flag value to LOGICAL to use logical bytes for storage billing, or to PHYSICAL to use physical bytes instead. LOGICAL is the default if this flag isn't specified. varType: string - name: tables description: A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels. From aec2db5ec0a9e3b3255b986bcc0f86de65f2287a Mon Sep 17 00:00:00 2001 From: SamuZad <47607001+SamuZad@users.noreply.github.com> Date: Thu, 1 Aug 2024 08:44:33 +0200 Subject: [PATCH 4/4] Update variables.tf Co-authored-by: Awais Malik --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index daed0f53..0f4d11af 100644 --- a/variables.tf +++ b/variables.tf @@ -56,7 +56,7 @@ variable "default_table_expiration_ms" { } variable "default_partition_expiration_ms" { - description = "The default partition expiration for all partitioned tables in the dataset, in MS" + description = "The default partition expiration for all partitioned tables in the dataset, in msec" type = number default = null }