diff --git a/pkg/plugins/datadog/cmd/main/main.go b/pkg/plugins/datadog/cmd/main/main.go index d84fdc9..7ae5b2e 100644 --- a/pkg/plugins/datadog/cmd/main/main.go +++ b/pkg/plugins/datadog/cmd/main/main.go @@ -340,6 +340,47 @@ func postProcess(ccResp *pb.CustomCostResponse) { // removes any items that have 0 usage, either because of post processing or otherwise ccResp.Costs = removeZeroUsages(ccResp.Costs) + + // DBM queries have 200 * number of hosts included. We need to adjust the costs to reflect this + ccResp.Costs = adjustDBMQueries(ccResp.Costs) +} + +// as per https://www.datadoghq.com/pricing/?product=database-monitoring#database-monitoring-can-i-still-use-dbm-if-i-have-additional-normalized-queries-past-the-a-hrefpricingallotmentsallotteda-amount +// the first 200 queries per host are free. +// if that zeroes out the dbm queries, we remove the cost +func adjustDBMQueries(costs []*pb.CustomCost) []*pb.CustomCost { + totalFreeQueries := float32(0.0) + for index := 0; index < len(costs); index++ { + if costs[index].ResourceName == "dbm_host_count" { + hostCount := costs[index].UsageQuantity + totalFreeQueries += 200 * float32(hostCount) + } + } + log.Debugf("total free queries: %f", totalFreeQueries) + + for index := 0; index < len(costs); index++ { + if costs[index].ResourceName == "dbm_queries_count" { + costs[index].UsageQuantity -= totalFreeQueries + log.Debugf("adjusted dbm queries: %v", costs[index]) + } + + } + + for index := 0; index < len(costs); index++ { + if costs[index].ResourceName == "dbm_queries_count" { + if costs[index].UsageQuantity <= 0 { + log.Debugf("removing cost %s because it has 0 usage", costs[index].ProviderId) + costs = append(costs[:index], costs[index+1:]...) + index = 0 + } else { + // TODO else, multiply cost by the rate for extra queries + costs[index].ListCost = 0.0 + costs[index].ListUnitPrice = 0.0 + costs[index].UsageUnit = "queries" + } + } + } + return costs } // removes any items that have 0 usage, either because of post processing or otherwise @@ -471,6 +512,8 @@ var usageToPricingMap = map[string]string{ "opentelemetry_apm_host_count": "apm_hosts", "apm_fargate_count": "apm_hosts", + "dbm_host_count": "dbm", + "dbm_queries_count": "dbm_queries", "container_count": "containers", "container_count_excl_agent": "containers", "billable_ingested_bytes": "ingested_logs", @@ -503,6 +546,7 @@ var rateFamilies = map[string]int{ "infra_hosts": 730.0, "apm_hosts": 730.0, "containers": 730.0, + "dbm": 730.0, } func getListingInfo(window opencost.Window, productfamily string, usageType string, listPricing *datadogplugin.PricingInformation) (description string, usageUnit string, pricing float32, currency string) { @@ -518,7 +562,6 @@ func getListingInfo(window opencost.Window, productfamily string, usageType stri // if it isn't, then the family is the pricing key pricingKey = productfamily } - matchedPrice := false // search through the pricing for the right key for _, detail := range listPricing.Details { @@ -536,7 +579,7 @@ func getListingInfo(window opencost.Window, productfamily string, usageType stri if hourlyPriceDenominator, found := rateFamilies[pricingKey]; found { // adjust the pricing to fit the window duration pricingPerHour := float32(pricingFloat) / float32(hourlyPriceDenominator) - pricingPerWindow := pricingPerHour //* float32(window.Duration().Hours()) + pricingPerWindow := pricingPerHour usageUnit = strings.TrimSuffix(usageUnit, "s") usageUnit += " - hours" pricing = pricingPerWindow diff --git a/pkg/plugins/datadog/cmd/main/main_test.go b/pkg/plugins/datadog/cmd/main/main_test.go new file mode 100644 index 0000000..2042f99 --- /dev/null +++ b/pkg/plugins/datadog/cmd/main/main_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "os" + "testing" + "time" + + datadogplugin "github.com/opencost/opencost-plugins/pkg/plugins/datadog/datadogplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/util/timeutil" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestPricingFetch(t *testing.T) { + listPricing, err := scrapeDatadogPrices(url) + if err != nil { + t.Fatalf("failed to get pricing: %v", err) + } + fmt.Printf("got response: %v", listPricing) + if len(listPricing.Details) == 0 { + t.Fatalf("expected non zero pricing details") + } +} + +func TestGetCustomCosts(t *testing.T) { + // read necessary env vars. If any are missing, log warning and skip test + ddSite := os.Getenv("DD_SITE") + ddApiKey := os.Getenv("DD_API_KEY") + ddAppKey := os.Getenv("DD_APPLICATION_KEY") + + if ddSite == "" { + log.Warnf("DD_SITE undefined, this needs to have the URL of your DD instance, skipping test") + t.Skip() + return + } + + if ddApiKey == "" { + log.Warnf("DD_API_KEY undefined, skipping test") + t.Skip() + return + } + + if ddAppKey == "" { + log.Warnf("DD_APPLICATION_KEY undefined, skipping test") + t.Skip() + return + } + + // write out config to temp file using contents of env vars + config := datadogplugin.DatadogConfig{ + DDSite: ddSite, + DDAPIKey: ddApiKey, + DDAppKey: ddAppKey, + } + + rateLimiter := rate.NewLimiter(0.25, 5) + ddCostSrc := DatadogCostSource{ + rateLimiter: rateLimiter, + } + ddCostSrc.ddCtx, ddCostSrc.usageApi = getDatadogClients(config) + windowStart := time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC) + // query for qty 2 of 1 hour windows + windowEnd := time.Date(2024, 10, 7, 0, 0, 0, 0, time.UTC) + + req := &pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(timeutil.Day), + } + + log.SetLogLevel("debug") + resp := ddCostSrc.GetCustomCosts(req) + + if len(resp) == 0 { + t.Fatalf("empty response") + } +} diff --git a/pkg/plugins/datadog/cmd/validator/main/main.go b/pkg/plugins/datadog/cmd/validator/main/main.go index dbf05a0..2bd9f39 100644 --- a/pkg/plugins/datadog/cmd/validator/main/main.go +++ b/pkg/plugins/datadog/cmd/validator/main/main.go @@ -102,16 +102,36 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { return false } + dbmCostsInRange := 0 //verify that the returned costs are non zero for _, resp := range respDaily { var costSum float32 for _, cost := range resp.Costs { costSum += cost.GetListCost() + if cost.GetListCost() > 100 { + log.Errorf("daily cost returned by plugin datadog for %v is greater than 100", cost) + return false + } + + //as of 10/2024, dbm hosts cost $84 a month or about $2.70. confirm that + // range + if cost.GetResourceName() == "dbm_host_count" { + // filter out recent costs since those might not be full days worth + if cost.GetListCost() > 2.5 && cost.GetListCost() < 3.0 { + dbmCostsInRange++ + } + } } if costSum == 0 { log.Errorf("daily costs returned by datadog plugin are zero") return false } + + } + + if dbmCostsInRange == 0 { + log.Errorf("no dbm costs in expected range found in daily costs") + return false } seenCosts := map[string]bool{} @@ -130,6 +150,7 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { "logs_indexed_events_15_day_count", "container_count_excl_agent", "agent_container", + "dbm_host_count", } for _, cost := range expectedCosts { @@ -141,6 +162,10 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { if len(seenCosts) != len(expectedCosts) { log.Errorf("hourly costs returned by plugin datadog do not equal expected costs") + log.Errorf("seen costs: %v", seenCosts) + log.Errorf("expected costs: %v", expectedCosts) + + log.Errorf("response: %v", respHourly) return false } @@ -153,11 +178,15 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { } seenCosts = map[string]bool{} - for _, resp := range respDaily { + for _, resp := range respHourly { for _, cost := range resp.Costs { seenCosts[cost.GetResourceName()] = true if cost.GetListCost() == 0 { - log.Errorf("daily cost returned by plugin datadog is zero") + log.Errorf("hourly cost returned by plugin datadog is zero") + return false + } + if cost.GetListCost() > 100 { + log.Errorf("hourly cost returned by plugin datadog for %v is greater than 100", cost) return false } } diff --git a/pkg/test/pkg/executor/main/main.go b/pkg/test/pkg/executor/main/main.go index 1a8ad31..b9c6a14 100644 --- a/pkg/test/pkg/executor/main/main.go +++ b/pkg/test/pkg/executor/main/main.go @@ -139,7 +139,7 @@ func invokeValidator(validatorPath, hourlyPath, dailyPath string) error { // invoke validator // Create the command with the given arguments - cmd := exec.Command("go", "run", validatorPath, hourlyPath, dailyPath) + cmd := exec.Command("go", "run", validatorPath, dailyPath, hourlyPath) // Run the command and capture the output output, err := cmd.CombinedOutput()