From 61fef7c42a7737f1ce805430553a7115e33bcabe Mon Sep 17 00:00:00 2001 From: Michael Denomy Date: Tue, 5 Dec 2023 11:36:38 -0500 Subject: [PATCH 1/3] Add support for New Relic OTLP endpoint --- docs/resources/service_vcl.md | 19 ++ ...ock_fastly_service_logging_newrelicotlp.go | 264 ++++++++++++++++++ ...astly_service_logging_newrelicotlp_test.go | 246 ++++++++++++++++ fastly/resource_fastly_service_vcl.go | 1 + 4 files changed, 530 insertions(+) create mode 100644 fastly/block_fastly_service_logging_newrelicotlp.go create mode 100644 fastly/block_fastly_service_logging_newrelicotlp_test.go diff --git a/docs/resources/service_vcl.md b/docs/resources/service_vcl.md index 2069a7e3b..e3615f5b4 100644 --- a/docs/resources/service_vcl.md +++ b/docs/resources/service_vcl.md @@ -278,6 +278,7 @@ $ terraform import fastly_service_vcl.demo xxxxxxxxxxxxxxxxxxxx@2 - `logging_loggly` (Block Set) (see [below for nested schema](#nestedblock--logging_loggly)) - `logging_logshuttle` (Block Set) (see [below for nested schema](#nestedblock--logging_logshuttle)) - `logging_newrelic` (Block Set) (see [below for nested schema](#nestedblock--logging_newrelic)) +- `logging_newrelicotlp` (Block Set) (see [below for nested schema](#nestedblock--logging_newrelicotlp)) - `logging_openstack` (Block Set) (see [below for nested schema](#nestedblock--logging_openstack)) - `logging_papertrail` (Block Set) (see [below for nested schema](#nestedblock--logging_papertrail)) - `logging_s3` (Block Set) (see [below for nested schema](#nestedblock--logging_s3)) @@ -902,6 +903,24 @@ Optional: - `response_condition` (String) The name of the condition to apply. + +### Nested Schema for `logging_newrelicotlp` + +Required: + +- `name` (String) The unique name of the New Relic OTLP logging endpoint. It is important to note that changing this attribute will delete and recreate the resource +- `token` (String, Sensitive) The Insert API key from the Account page of your New Relic account + +Optional: + +- `format` (String) Apache style log formatting. Your log must produce valid JSON that New Relic OTLP can ingest. +- `format_version` (Number) The version of the custom logging format used for the configured endpoint. Can be either `1` or `2`. (default: `2`). +- `placement` (String) Where in the generated VCL the logging call should be placed. +- `region` (String) The region that log data will be sent to. Default: `US` +- `url` (String) The optional URL of a New Relic trace observer to send logs to. Must be a New Relic domain name. +- `response_condition` (String) The name of the condition to apply. + + ### Nested Schema for `logging_openstack` diff --git a/fastly/block_fastly_service_logging_newrelicotlp.go b/fastly/block_fastly_service_logging_newrelicotlp.go new file mode 100644 index 000000000..5c97b810f --- /dev/null +++ b/fastly/block_fastly_service_logging_newrelicotlp.go @@ -0,0 +1,264 @@ +package fastly + +import ( + "context" + "fmt" + "log" + + gofastly "github.com/fastly/go-fastly/v8/fastly" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +// NewRelicOTLPServiceAttributeHandler provides a base implementation for ServiceAttributeDefinition. +type NewRelicOTLPServiceAttributeHandler struct { + *DefaultServiceAttributeHandler +} + +// NewServiceLoggingNewRelicOTLP returns a new resource. +func NewServiceLoggingNewRelicOTLP(sa ServiceMetadata) ServiceAttributeDefinition { + return ToServiceAttributeDefinition(&NewRelicOTLPServiceAttributeHandler{ + &DefaultServiceAttributeHandler{ + key: "logging_newrelicotlp", + serviceMetadata: sa, + }, + }) +} + +// Key returns the resource key. +func (h *NewRelicOTLPServiceAttributeHandler) Key() string { + return h.key +} + +// GetSchema returns the resource schema. +func (h *NewRelicOTLPServiceAttributeHandler) GetSchema() *schema.Schema { + blockAttributes := map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + Description: "The unique name of the New Relic OTLP logging endpoint. It is important to note that changing this attribute will delete and recreate the resource", + }, + "region": { + Type: schema.TypeString, + Optional: true, + Default: "US", + Description: "The region that log data will be sent to. Default: `US`", + }, + "token": { + Type: schema.TypeString, + Required: true, + Sensitive: true, + Description: "The Insert API key from the Account page of your New Relic account", + }, + } + + if h.GetServiceMetadata().serviceType == ServiceTypeVCL { + blockAttributes["format"] = &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "Apache style log formatting. Your log must produce valid JSON that New Relic Logs can ingest.", + } + blockAttributes["format_version"] = &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 2, + Description: "The version of the custom logging format used for the configured endpoint. Can be either `1` or `2`. (default: `2`).", + ValidateDiagFunc: validateLoggingFormatVersion(), + } + blockAttributes["placement"] = &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "Where in the generated VCL the logging call should be placed.", + ValidateDiagFunc: validateLoggingPlacement(), + } + blockAttributes["response_condition"] = &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "The name of the condition to apply.", + } + blockAttributes["url"] = &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "The optional New Relic Trace Observer URL to stream logs to.", + } + } + + return &schema.Schema{ + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Resource{ + Schema: blockAttributes, + }, + } +} + +// Create creates the resource. +func (h *NewRelicOTLPServiceAttributeHandler) Create(_ context.Context, d *schema.ResourceData, resource map[string]any, serviceVersion int, conn *gofastly.Client) error { + opts := h.buildCreate(resource, d.Id(), serviceVersion) + + log.Printf("[DEBUG] Fastly New Relic OTLP logging addition opts: %#v", opts) + + return createNewRelicOTLP(conn, opts) +} + +// Read refreshes the resource. +func (h *NewRelicOTLPServiceAttributeHandler) Read(_ context.Context, d *schema.ResourceData, _ map[string]any, serviceVersion int, conn *gofastly.Client) error { + localState := d.Get(h.GetKey()).(*schema.Set).List() + + if len(localState) > 0 || d.Get("imported").(bool) || d.Get("force_refresh").(bool) { + log.Printf("[DEBUG] Refreshing New Relic OTLP logging endpoints for (%s)", d.Id()) + remoteState, err := conn.ListNewRelicOTLP(&gofastly.ListNewRelicOTLPInput{ + ServiceID: d.Id(), + ServiceVersion: serviceVersion, + }) + if err != nil { + return fmt.Errorf("error looking up New Relic OTLP logging endpoints for (%s), version (%v): %s", d.Id(), serviceVersion, err) + } + + dll := flattenNewRelicOTLP(remoteState) + + for _, element := range dll { + h.pruneVCLLoggingAttributes(element) + } + + if err := d.Set(h.GetKey(), dll); err != nil { + log.Printf("[WARN] Error setting New Relic OTLP logging endpoints for (%s): %s", d.Id(), err) + } + } + + return nil +} + +// Update updates the resource. +func (h *NewRelicOTLPServiceAttributeHandler) Update(_ context.Context, d *schema.ResourceData, resource, modified map[string]any, serviceVersion int, conn *gofastly.Client) error { + opts := gofastly.UpdateNewRelicOTLPInput{ + ServiceID: d.Id(), + ServiceVersion: serviceVersion, + Name: resource["name"].(string), + } + + // NOTE: When converting from an interface{} we lose the underlying type. + // Converting to the wrong type will result in a runtime panic. + if v, ok := modified["token"]; ok { + opts.Token = gofastly.String(v.(string)) + } + if v, ok := modified["format"]; ok { + opts.Format = gofastly.String(v.(string)) + } + if v, ok := modified["format_version"]; ok { + opts.FormatVersion = gofastly.Int(v.(int)) + } + if v, ok := modified["response_condition"]; ok { + opts.ResponseCondition = gofastly.String(v.(string)) + } + if v, ok := modified["placement"]; ok { + opts.Placement = gofastly.String(v.(string)) + } + if v, ok := modified["region"]; ok { + opts.Region = gofastly.String(v.(string)) + } + if v, ok := modified["url"]; ok { + opts.URL = gofastly.String(v.(string)) + } + + log.Printf("[DEBUG] Update New Relic OTLP Opts: %#v", opts) + _, err := conn.UpdateNewRelicOTLP(&opts) + if err != nil { + return err + } + return nil +} + +// Delete deletes the resource. +func (h *NewRelicOTLPServiceAttributeHandler) Delete(_ context.Context, d *schema.ResourceData, resource map[string]any, serviceVersion int, conn *gofastly.Client) error { + opts := h.buildDelete(resource, d.Id(), serviceVersion) + + log.Printf("[DEBUG] Fastly New Relic OTLP logging endpoint removal opts: %#v", opts) + + return deleteNewRelicOTLP(conn, opts) +} + +func createNewRelicOTLP(conn *gofastly.Client, i *gofastly.CreateNewRelicOTLPInput) error { + _, err := conn.CreateNewRelicOTLP(i) + return err +} + +func deleteNewRelicOTLP(conn *gofastly.Client, i *gofastly.DeleteNewRelicOTLPInput) error { + err := conn.DeleteNewRelicOTLP(i) + + errRes, ok := err.(*gofastly.HTTPError) + if !ok { + return err + } + // 404 response codes don't result in an error propagating because a 404 could + // indicate that a resource was deleted elsewhere. + if !errRes.IsNotFound() { + return err + } + return nil +} + +// flattenNewRelicOTLP models data into format suitable for saving to Terraform state. +func flattenNewRelicOTLP(remoteState []*gofastly.NewRelicOTLP) []map[string]any { + var result []map[string]any + for _, resource := range remoteState { + data := map[string]any{ + "name": resource.Name, + "token": resource.Token, + "format": resource.Format, + "format_version": resource.FormatVersion, + "placement": resource.Placement, + "region": resource.Region, + "response_condition": resource.ResponseCondition, + "url": resource.URL, + } + + // Prune any empty values that come from the default string value in structs. + for k, v := range data { + if v == "" { + delete(data, k) + } + } + + result = append(result, data) + } + + return result +} + +func (h *NewRelicOTLPServiceAttributeHandler) buildCreate(newrelicotlpMap any, serviceID string, serviceVersion int) *gofastly.CreateNewRelicOTLPInput { + resource := newrelicotlpMap.(map[string]any) + + vla := h.getVCLLoggingAttributes(resource) + opts := &gofastly.CreateNewRelicOTLPInput{ + Format: gofastly.String(vla.format), + FormatVersion: vla.formatVersion, + Name: gofastly.String(resource["name"].(string)), + Region: gofastly.String(resource["region"].(string)), + ServiceID: serviceID, + ServiceVersion: serviceVersion, + Token: gofastly.String(resource["token"].(string)), + URL: gofastly.String(resource["url"].(string)), + } + + // WARNING: The following fields shouldn't have an empty string passed. + // As it will cause the Fastly API to return an error. + // This is because go-fastly v7+ will not 'omitempty' due to pointer type. + if vla.placement != "" { + opts.Placement = gofastly.String(vla.placement) + } + if vla.responseCondition != "" { + opts.ResponseCondition = gofastly.String(vla.responseCondition) + } + + return opts +} + +func (h *NewRelicOTLPServiceAttributeHandler) buildDelete(newrelicotlpMap any, serviceID string, serviceVersion int) *gofastly.DeleteNewRelicOTLPInput { + resource := newrelicotlpMap.(map[string]any) + + return &gofastly.DeleteNewRelicOTLPInput{ + ServiceID: serviceID, + ServiceVersion: serviceVersion, + Name: resource["name"].(string), + } +} diff --git a/fastly/block_fastly_service_logging_newrelicotlp_test.go b/fastly/block_fastly_service_logging_newrelicotlp_test.go new file mode 100644 index 000000000..a9a1ebbb4 --- /dev/null +++ b/fastly/block_fastly_service_logging_newrelicotlp_test.go @@ -0,0 +1,246 @@ +package fastly + +import ( + "fmt" + "log" + "testing" + + gofastly "github.com/fastly/go-fastly/v8/fastly" + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +func TestResourceFastlyFlattenNewRelicOTLP(t *testing.T) { + cases := []struct { + remote []*gofastly.NewRelicOTLP + local []map[string]any + }{ + { + remote: []*gofastly.NewRelicOTLP{ + { + ServiceVersion: 1, + Name: "newrelicotlp-endpoint", + Token: "token", + Region: "US", + FormatVersion: 2, + }, + }, + local: []map[string]any{ + { + "name": "newrelicotlp-endpoint", + "token": "token", + "region": "US", + "format_version": 2, + }, + }, + }, + } + + for _, c := range cases { + out := flattenNewRelicOTLP(c.remote) + if diff := cmp.Diff(out, c.local); diff != "" { + t.Fatalf("Error matching: %s", diff) + } + } +} + +var newrelicotlpDefaultFormat = `{ + "time_elapsed":%{time.elapsed.usec}V, + "is_tls":%{if(req.is_ssl, "true", "false")}V, + "client_ip":"%{req.http.Fastly-Client-IP}V", + "geo_city":"%{client.geo.city}V", + "geo_country_code":"%{client.geo.country_code}V", + "request":"%{req.request}V", + "host":"%{req.http.Fastly-Orig-Host}V", + "url":"%{json.escape(req.url)}V", + "request_referer":"%{json.escape(req.http.Referer)}V", + "request_user_agent":"%{json.escape(req.http.User-Agent)}V", + "request_accept_language":"%{json.escape(req.http.Accept-Language)}V", + "request_accept_charset":"%{json.escape(req.http.Accept-Charset)}V", + "cache_status":"%{regsub(fastly_info.state, "^(HIT-(SYNTH)|(HITPASS|HIT|MISS|PASS|ERROR|PIPE)).*", "\2\3") }V" +}` + +func TestAccFastlyServiceVCL_logging_newrelicotlp_basic(t *testing.T) { + var service gofastly.ServiceDetail + name := fmt.Sprintf("tf-test-%s", acctest.RandString(10)) + domain := fmt.Sprintf("fastly-test.%s.com", name) + + log1 := gofastly.NewRelicOTLP{ + ServiceVersion: 1, + Name: "newrelicotlp-endpoint", + Token: "token", + Region: "US", + FormatVersion: 2, + Format: "%h %l %u %t \"%r\" %>s %b", + } + + log1AfterUpdate := gofastly.NewRelicOTLP{ + ServiceVersion: 1, + Name: "newrelicotlp-endpoint", + Token: "t0k3n", + Region: "EU", + FormatVersion: 2, + Format: "%h %l %u %t \"%r\" %>s %b %T", + } + + log2 := gofastly.NewRelicOTLP{ + ServiceVersion: 1, + Name: "another-newrelicotlp-endpoint", + Token: "another-token", + Region: "US", + URL: "https://example.nr-data.net", + FormatVersion: 2, + Format: appendNewLine(newrelicotlpDefaultFormat), + } + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheck(t) + }, + ProviderFactories: testAccProviders, + CheckDestroy: testAccCheckServiceVCLDestroy, + Steps: []resource.TestStep{ + { + Config: testAccServiceVCLNewRelicOTLPConfig(name, domain), + Check: resource.ComposeTestCheckFunc( + testAccCheckServiceExists("fastly_service_vcl.foo", &service), + testAccCheckFastlyServiceVCLNewRelicOTLPAttributes(&service, []*gofastly.NewRelicOTLP{&log1}, ServiceTypeVCL), + resource.TestCheckResourceAttr( + "fastly_service_vcl.foo", "name", name), + resource.TestCheckResourceAttr( + "fastly_service_vcl.foo", "logging_newrelicotlp.#", "1"), + ), + }, + + { + Config: testAccServiceVCLNewRelicOTLPConfigUpdate(name, domain), + Check: resource.ComposeTestCheckFunc( + testAccCheckServiceExists("fastly_service_vcl.foo", &service), + testAccCheckFastlyServiceVCLNewRelicOTLPAttributes(&service, []*gofastly.NewRelicOTLP{&log1AfterUpdate, &log2}, ServiceTypeVCL), + resource.TestCheckResourceAttr( + "fastly_service_vcl.foo", "name", name), + resource.TestCheckResourceAttr( + "fastly_service_vcl.foo", "logging_newrelicotlp.#", "2"), + ), + PreventDiskCleanup: true, + }, + }, + }) +} + +func testAccCheckFastlyServiceVCLNewRelicOTLPAttributes(service *gofastly.ServiceDetail, newrelicotlp []*gofastly.NewRelicOTLP, serviceType string) resource.TestCheckFunc { + return func(_ *terraform.State) error { + conn := testAccProvider.Meta().(*APIClient).conn + newrelicotlpList, err := conn.ListNewRelicOTLP(&gofastly.ListNewRelicOTLPInput{ + ServiceID: service.ID, + ServiceVersion: service.ActiveVersion.Number, + }) + if err != nil { + return fmt.Errorf("error looking up NewRelic OTLP Logging for (%s), version (%d): %s", service.Name, service.ActiveVersion.Number, err) + } + + if len(newrelicotlpList) != len(newrelicotlp) { + return fmt.Errorf("newRelic List count mismatch, expected (%d), got (%d)", len(newrelicotlp), len(newrelicotlpList)) + } + + log.Printf("[DEBUG] newrelicotlpList = %#v\n", newrelicotlpList) + + var found int + for _, d := range newrelicotlp { + for _, dl := range newrelicotlpList { + if d.Name == dl.Name { + // we don't know these things ahead of time, so populate them now + d.ServiceID = service.ID + d.ServiceVersion = service.ActiveVersion.Number + // We don't track these, so clear them out because we also won't know + // these ahead of time + dl.CreatedAt = nil + dl.UpdatedAt = nil + + // Ignore VCL attributes for Compute and set to whatever is returned from the API. + if serviceType == ServiceTypeCompute { + dl.FormatVersion = d.FormatVersion + dl.Format = d.Format + dl.ResponseCondition = d.ResponseCondition + dl.Placement = d.Placement + } + + if diff := cmp.Diff(d, dl); diff != "" { + return fmt.Errorf("bad match NewRelic OTLP logging match: %s", diff) + } + found++ + } + } + } + + if found != len(newrelicotlp) { + return fmt.Errorf("error matching NewRelic OTLP Logging rules") + } + + return nil + } +} + +func testAccServiceVCLNewRelicOTLPConfig(name string, domain string) string { + return fmt.Sprintf(` +resource "fastly_service_vcl" "foo" { + name = "%s" + + domain { + name = "%s" + comment = "tf-newrelicotlp-logging" + } + + backend { + address = "aws.amazon.com" + name = "amazon docs" + } + + logging_newrelicotlp { + name = "newrelicotlp-endpoint" + token = "token" + format = "%%h %%l %%u %%t \"%%r\" %%>s %%b" + } + + force_destroy = true +} +`, name, domain) +} + +func testAccServiceVCLNewRelicOTLPConfigUpdate(name, domain string) string { + return fmt.Sprintf(` +resource "fastly_service_vcl" "foo" { + name = "%s" + + domain { + name = "%s" + comment = "tf-newrelicotlp-logging" + } + + backend { + address = "aws.amazon.com" + name = "amazon docs" + } + + logging_newrelicotlp { + name = "newrelicotlp-endpoint" + token = "t0k3n" + format = "%%h %%l %%u %%t \"%%r\" %%>s %%b %%T" + region = "EU" +} + + logging_newrelicotlp { + name = "another-newrelicotlp-endpoint" + token = "another-token" + url = "https://example.nr-data.net" + format = < Date: Tue, 5 Dec 2023 13:09:48 -0500 Subject: [PATCH 2/3] Update docs --- docs/resources/service_vcl.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/resources/service_vcl.md b/docs/resources/service_vcl.md index e3615f5b4..955495626 100644 --- a/docs/resources/service_vcl.md +++ b/docs/resources/service_vcl.md @@ -913,12 +913,12 @@ Required: Optional: -- `format` (String) Apache style log formatting. Your log must produce valid JSON that New Relic OTLP can ingest. +- `format` (String) Apache style log formatting. Your log must produce valid JSON that New Relic Logs can ingest. - `format_version` (Number) The version of the custom logging format used for the configured endpoint. Can be either `1` or `2`. (default: `2`). - `placement` (String) Where in the generated VCL the logging call should be placed. - `region` (String) The region that log data will be sent to. Default: `US` -- `url` (String) The optional URL of a New Relic trace observer to send logs to. Must be a New Relic domain name. - `response_condition` (String) The name of the condition to apply. +- `url` (String) The New Relic Trace Observer URL to stream logs to for New Relic Infinite Tracing. From f562f4a8468da1c9c65a98449ed69e60ee3a8b8a Mon Sep 17 00:00:00 2001 From: Michael Denomy Date: Tue, 5 Dec 2023 13:34:44 -0500 Subject: [PATCH 3/3] Fix docs --- docs/resources/service_vcl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/resources/service_vcl.md b/docs/resources/service_vcl.md index 955495626..8aafbcf30 100644 --- a/docs/resources/service_vcl.md +++ b/docs/resources/service_vcl.md @@ -918,7 +918,7 @@ Optional: - `placement` (String) Where in the generated VCL the logging call should be placed. - `region` (String) The region that log data will be sent to. Default: `US` - `response_condition` (String) The name of the condition to apply. -- `url` (String) The New Relic Trace Observer URL to stream logs to for New Relic Infinite Tracing. +- `url` (String) The optional New Relic Trace Observer URL to stream logs to.