[DATADOG][ENG-2797] Support DD DBM hosts/queries (#36)

* use deploy key Signed-off-by: Alex Meijer <[email protected]> * clean up, try out webhook Signed-off-by: Alex Meijer <[email protected]> * bugfixes Signed-off-by: Alex Meijer <[email protected]> * last round of tweaks Signed-off-by: Alex Meijer <[email protected]> * [DATADOG] [ENG-2797] Improved Support for DBM costs Signed-off-by: Alex Meijer <[email protected]> * bugfixes Signed-off-by: Alex Meijer <[email protected]> --------- Signed-off-by: Alex Meijer <[email protected]>
opencost · Oct 8, 2024 · 30e43a6 · 30e43a6
1 parent 7acbbf3
commit 30e43a6
Show file tree

Hide file tree

Showing 4 changed files with 158 additions and 5 deletions.
diff --git a/pkg/plugins/datadog/cmd/main/main.go b/pkg/plugins/datadog/cmd/main/main.go
@@ -340,6 +340,47 @@ func postProcess(ccResp *pb.CustomCostResponse) {
 
 	// removes any items that have 0 usage, either because of post processing or otherwise
 	ccResp.Costs = removeZeroUsages(ccResp.Costs)
+
+	// DBM queries have 200 * number of hosts included. We need to adjust the costs to reflect this
+	ccResp.Costs = adjustDBMQueries(ccResp.Costs)
+}
+
+// as per https://www.datadoghq.com/pricing/?product=database-monitoring#database-monitoring-can-i-still-use-dbm-if-i-have-additional-normalized-queries-past-the-a-hrefpricingallotmentsallotteda-amount
+// the first 200 queries per host are free.
+// if that zeroes out the dbm queries, we remove the cost
+func adjustDBMQueries(costs []*pb.CustomCost) []*pb.CustomCost {
+	totalFreeQueries := float32(0.0)
+	for index := 0; index < len(costs); index++ {
+		if costs[index].ResourceName == "dbm_host_count" {
+			hostCount := costs[index].UsageQuantity
+			totalFreeQueries += 200 * float32(hostCount)
+		}
+	}
+	log.Debugf("total free queries: %f", totalFreeQueries)
+
+	for index := 0; index < len(costs); index++ {
+		if costs[index].ResourceName == "dbm_queries_count" {
+			costs[index].UsageQuantity -= totalFreeQueries
+			log.Debugf("adjusted dbm queries: %v", costs[index])
+		}
+
+	}
+
+	for index := 0; index < len(costs); index++ {
+		if costs[index].ResourceName == "dbm_queries_count" {
+			if costs[index].UsageQuantity <= 0 {
+				log.Debugf("removing cost %s because it has 0 usage", costs[index].ProviderId)
+				costs = append(costs[:index], costs[index+1:]...)
+				index = 0
+			} else {
+				// TODO else, multiply cost by the rate for extra queries
+				costs[index].ListCost = 0.0
+				costs[index].ListUnitPrice = 0.0
+				costs[index].UsageUnit = "queries"
+			}
+		}
+	}
+	return costs
 }
 
 // removes any items that have 0 usage, either because of post processing or otherwise
@@ -471,6 +512,8 @@ var usageToPricingMap = map[string]string{
 	"opentelemetry_apm_host_count":     "apm_hosts",
 	"apm_fargate_count":                "apm_hosts",
 
+	"dbm_host_count":                 "dbm",
+	"dbm_queries_count":              "dbm_queries",
 	"container_count":                "containers",
 	"container_count_excl_agent":     "containers",
 	"billable_ingested_bytes":        "ingested_logs",
@@ -503,6 +546,7 @@ var rateFamilies = map[string]int{
 	"infra_hosts": 730.0,
 	"apm_hosts":   730.0,
 	"containers":  730.0,
+	"dbm":         730.0,
 }
 
 func getListingInfo(window opencost.Window, productfamily string, usageType string, listPricing *datadogplugin.PricingInformation) (description string, usageUnit string, pricing float32, currency string) {
@@ -518,7 +562,6 @@ func getListingInfo(window opencost.Window, productfamily string, usageType stri
 		// if it isn't, then the family is the pricing key
 		pricingKey = productfamily
 	}
-
 	matchedPrice := false
 	// search through the pricing for the right key
 	for _, detail := range listPricing.Details {
@@ -536,7 +579,7 @@ func getListingInfo(window opencost.Window, productfamily string, usageType stri
 			if hourlyPriceDenominator, found := rateFamilies[pricingKey]; found {
 				// adjust the pricing to fit the window duration
 				pricingPerHour := float32(pricingFloat) / float32(hourlyPriceDenominator)
-				pricingPerWindow := pricingPerHour //* float32(window.Duration().Hours())
+				pricingPerWindow := pricingPerHour
 				usageUnit = strings.TrimSuffix(usageUnit, "s")
 				usageUnit += " - hours"
 				pricing = pricingPerWindow

diff --git a/pkg/plugins/datadog/cmd/main/main_test.go b/pkg/plugins/datadog/cmd/main/main_test.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	datadogplugin "github.com/opencost/opencost-plugins/pkg/plugins/datadog/datadogplugin"
+	"github.com/opencost/opencost/core/pkg/log"
+	"github.com/opencost/opencost/core/pkg/model/pb"
+	"github.com/opencost/opencost/core/pkg/util/timeutil"
+	"golang.org/x/time/rate"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestPricingFetch(t *testing.T) {
+	listPricing, err := scrapeDatadogPrices(url)
+	if err != nil {
+		t.Fatalf("failed to get pricing: %v", err)
+	}
+	fmt.Printf("got response: %v", listPricing)
+	if len(listPricing.Details) == 0 {
+		t.Fatalf("expected non zero pricing details")
+	}
+}
+
+func TestGetCustomCosts(t *testing.T) {
+	// read necessary env vars. If any are missing, log warning and skip test
+	ddSite := os.Getenv("DD_SITE")
+	ddApiKey := os.Getenv("DD_API_KEY")
+	ddAppKey := os.Getenv("DD_APPLICATION_KEY")
+
+	if ddSite == "" {
+		log.Warnf("DD_SITE undefined, this needs to have the URL of your DD instance, skipping test")
+		t.Skip()
+		return
+	}
+
+	if ddApiKey == "" {
+		log.Warnf("DD_API_KEY undefined, skipping test")
+		t.Skip()
+		return
+	}
+
+	if ddAppKey == "" {
+		log.Warnf("DD_APPLICATION_KEY undefined, skipping test")
+		t.Skip()
+		return
+	}
+
+	// write out config to temp file using contents of env vars
+	config := datadogplugin.DatadogConfig{
+		DDSite:   ddSite,
+		DDAPIKey: ddApiKey,
+		DDAppKey: ddAppKey,
+	}
+
+	rateLimiter := rate.NewLimiter(0.25, 5)
+	ddCostSrc := DatadogCostSource{
+		rateLimiter: rateLimiter,
+	}
+	ddCostSrc.ddCtx, ddCostSrc.usageApi = getDatadogClients(config)
+	windowStart := time.Date(2024, 10, 6, 0, 0, 0, 0, time.UTC)
+	// query for qty 2 of 1 hour windows
+	windowEnd := time.Date(2024, 10, 7, 0, 0, 0, 0, time.UTC)
+
+	req := &pb.CustomCostRequest{
+		Start:      timestamppb.New(windowStart),
+		End:        timestamppb.New(windowEnd),
+		Resolution: durationpb.New(timeutil.Day),
+	}
+
+	log.SetLogLevel("debug")
+	resp := ddCostSrc.GetCustomCosts(req)
+
+	if len(resp) == 0 {
+		t.Fatalf("empty response")
+	}
+}
diff --git a/pkg/plugins/datadog/cmd/validator/main/main.go b/pkg/plugins/datadog/cmd/validator/main/main.go
@@ -102,16 +102,36 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool {
 		return false
 	}
 
+	dbmCostsInRange := 0
 	//verify that the returned costs are non zero
 	for _, resp := range respDaily {
 		var costSum float32
 		for _, cost := range resp.Costs {
 			costSum += cost.GetListCost()
+			if cost.GetListCost() > 100 {
+				log.Errorf("daily cost returned by plugin datadog for %v is greater than 100", cost)
+				return false
+			}
+
+			//as of 10/2024, dbm hosts cost $84 a month or about $2.70. confirm that
+			// range
+			if cost.GetResourceName() == "dbm_host_count" {
+				// filter out recent costs since those might not be full days worth
+				if cost.GetListCost() > 2.5 && cost.GetListCost() < 3.0 {
+					dbmCostsInRange++
+				}
+			}
 		}
 		if costSum == 0 {
 			log.Errorf("daily costs returned by datadog plugin are zero")
 			return false
 		}
+
+	}
+
+	if dbmCostsInRange == 0 {
+		log.Errorf("no dbm costs in expected range found in daily costs")
+		return false
 	}
 
 	seenCosts := map[string]bool{}
@@ -130,6 +150,7 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool {
 		"logs_indexed_events_15_day_count",
 		"container_count_excl_agent",
 		"agent_container",
+		"dbm_host_count",
 	}
 
 	for _, cost := range expectedCosts {
@@ -141,6 +162,10 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool {
 
 	if len(seenCosts) != len(expectedCosts) {
 		log.Errorf("hourly costs returned by plugin datadog do not equal expected costs")
+		log.Errorf("seen costs: %v", seenCosts)
+		log.Errorf("expected costs: %v", expectedCosts)
+
+		log.Errorf("response: %v", respHourly)
 		return false
 	}
 
@@ -153,11 +178,15 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool {
 	}
 
 	seenCosts = map[string]bool{}
-	for _, resp := range respDaily {
+	for _, resp := range respHourly {
 		for _, cost := range resp.Costs {
 			seenCosts[cost.GetResourceName()] = true
 			if cost.GetListCost() == 0 {
-				log.Errorf("daily cost returned by plugin datadog is zero")
+				log.Errorf("hourly cost returned by plugin datadog is zero")
+				return false
+			}
+			if cost.GetListCost() > 100 {
+				log.Errorf("hourly cost returned by plugin datadog for %v is greater than 100", cost)
 				return false
 			}
 		}

diff --git a/pkg/test/pkg/executor/main/main.go b/pkg/test/pkg/executor/main/main.go
@@ -139,7 +139,7 @@ func invokeValidator(validatorPath, hourlyPath, dailyPath string) error {
 	// invoke validator
 
 	// Create the command with the given arguments
-	cmd := exec.Command("go", "run", validatorPath, hourlyPath, dailyPath)
+	cmd := exec.Command("go", "run", validatorPath, dailyPath, hourlyPath)
 
 	// Run the command and capture the output
 	output, err := cmd.CombinedOutput()