Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Fix Cost Calculation for AWS CloudFront Distributions #1083

Closed
wants to merge 5 commits into from
Closed

feat: Fix Cost Calculation for AWS CloudFront Distributions #1083

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e2c34bd
fixed cost of cloudfront
bishal7679 Oct 11, 2023
a96976f
added function invocation cost of cloudfront
bishal7679 Oct 12, 2023
bc4cc8a
Merge branch 'develop' into feature/correct-cost-for-cloudfront
bishal7679 Oct 12, 2023
258f4f3
Merge branch 'develop' into feature/correct-cost-for-cloudfront
bishal7679 Oct 12, 2023
8768cc3
Merge branch 'develop' into feature/correct-cost-for-cloudfront
bishal7679 Oct 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 182 additions & 10 deletions providers/aws/cloudfront/distributions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ package cloudfront

import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"

log "github.com/sirupsen/logrus"
Expand All @@ -11,11 +14,63 @@ import (
"github.com/aws/aws-sdk-go-v2/service/cloudfront"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
"github.com/aws/aws-sdk-go-v2/service/pricing"
pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types"
. "github.com/tailwarden/komiser/models"
. "github.com/tailwarden/komiser/providers"
awsUtils "github.com/tailwarden/komiser/providers/aws/utils"
"github.com/tailwarden/komiser/utils"
)

const (
freeTierRequests = 10000000
freeTierFunctionInvocation = 2000000
freeTierUpload = 1099511627776
)

func ConvertBytesToTerabytes(bytes int64) float64 {
return float64(bytes) / 1099511627776
}
Comment on lines +31 to +33
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we convert to GB instead? We do the same by multiplying by 1024 to the result

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I thought it would be more efficient not to multiply inline

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok i got it


func getRate(pricingOutput *pricing.GetProductsOutput, groupDescriptionNameOne string, groupDescriptionNameTwo string) (float64, error) {
costPerMonth := 0.0

if pricingOutput != nil && len(pricingOutput.PriceList) > 0 {
var priceList interface{}
err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &priceList)
if err != nil {
return 0, fmt.Errorf("failed to unmarshal JSON: %w", err)
}

priceListMap := priceList.(map[string]interface{})
if attribute, ok := priceListMap["product"].(map[string]interface{})["attributes"]; ok {
for _, attb := range attribute.(map[string]interface{}) {
if description, ok := attb.(map[string]string)["groupDescription"]; ok {
if strings.Contains(description, groupDescriptionNameOne) || strings.Contains(description, groupDescriptionNameTwo) {
if onDemand, ok := priceListMap["terms"].(map[string]interface{})["OnDemand"]; ok {
for _, details := range onDemand.(map[string]interface{}) {
if priceDetails, ok := details.(map[string]interface{})["priceDimensions"].(map[string]interface{}); ok {
for _, price := range priceDetails {
usdPrice := price.(map[string]interface{})["pricePerUnit"].(map[string]interface{})["USD"].(string)
costPerMonth, err = strconv.ParseFloat(usdPrice, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse cost per month: %w", err)
}
break
}
}
}
}
}
}
}
}

}

return costPerMonth, nil
}

func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) {
resources := make([]Resource, 0)
var config cloudfront.ListDistributionsInput
Expand All @@ -25,6 +80,28 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
client.AWSClient.Region = "us-east-1"
cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient)
client.AWSClient.Region = tempRegion
pricingClient := pricing.NewFromConfig(*client.AWSClient)

pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{
ServiceCode: aws.String("AmazonCloudFront"),
Filters: []pricingTypes.Filter{
{
Field: aws.String("regionCode"),
Value: aws.String(client.AWSClient.Region),
Type: pricingTypes.FilterTypeTermMatch,
},
},
})
if err != nil {
log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err)
return resources, err
}

priceMap, err := awsUtils.GetPriceMap(pricingOutput)
if err != nil {
log.Errorf("ERROR: Failed to calculate cost per month: %v", err)
return resources, err
}

for {
output, err := cloudfrontClient.ListDistributions(ctx, &config)
Expand All @@ -39,7 +116,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
MetricName: aws.String("BytesDownloaded"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -59,13 +136,15 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
bytesDownloaded = *metricsBytesDownloadedOutput.Datapoints[0].Sum
}

sizeInTBDownload := ConvertBytesToTerabytes(int64(bytesDownloaded))

metricsBytesUploadedOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("BytesUploaded"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -84,14 +163,19 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 {
bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum
}
if bytesUploaded > freeTierUpload {
bytesUploaded -= freeTierUpload
}

sizeInTBUpload := ConvertBytesToTerabytes(int64(bytesUploaded))

metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Requests"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -110,17 +194,105 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 {
requests = *metricsRequestsOutput.Datapoints[0].Sum
}
if requests > freeTierRequests {
requests -= freeTierRequests
}

metricsLambdaEdgeDurationOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Duration"),
Namespace: aws.String("AWS/LambdaEdge"),
Dimensions: []types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
},
Period: aws.Int32(86400),
Statistics: []types.Statistic{
types.StatisticAverage,
},
})

if err != nil {
log.Warnf("Couldn't fetch Lambda@Edge Duration metric for %s", *distribution.Id)
}

lambdaEdgeDuration := 0.0
if metricsLambdaEdgeDurationOutput != nil && len(metricsLambdaEdgeDurationOutput.Datapoints) > 0 {
lambdaEdgeDuration = *metricsLambdaEdgeDurationOutput.Datapoints[0].Average
}

metricsLambdaEdgeRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Requests"),
Namespace: aws.String("AWS/LambdaEdge"),
Dimensions: []types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
},
Period: aws.Int32(86400),
Statistics: []types.Statistic{
types.StatisticSum,
},
})

if err != nil {
log.Warnf("Couldn't fetch Lambda@Edge Requests metric for %s", *distribution.Id)
}

lambdaEdgeRequests := 0.0
if metricsLambdaEdgeRequestsOutput != nil && len(metricsLambdaEdgeRequestsOutput.Datapoints) > 0 {
lambdaEdgeRequests = *metricsLambdaEdgeRequestsOutput.Datapoints[0].Sum
}

metricsFunctionInvocationsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("FunctionInvocations"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
},
Period: aws.Int32(3600),
Statistics: []types.Statistic{
types.StatisticSum,
},
})
if err != nil {
log.Warnf("Couldn't fetch Function Invocations metric for %s", *distribution.Id)
}

functionInvocation := 0.0
if metricsFunctionInvocationsOutput != nil && len(metricsFunctionInvocationsOutput.Datapoints) > 0 {
functionInvocation = *metricsFunctionInvocationsOutput.Datapoints[0].Sum
}
if functionInvocation > freeTierFunctionInvocation {
functionInvocation -= freeTierFunctionInvocation
}

dataTransferToInternetCost := awsUtils.GetCost(priceMap["CloudFront-DataTransfer-In-Bytes"], sizeInTBUpload*1024)

dataTransferToOriginCost := awsUtils.GetCost(priceMap["CloudFront-DataTransfer-Out-Bytes"], sizeInTBDownload*1024)

requestsCost := awsUtils.GetCost(priceMap["CloudFront-Requests"], requests/10000)
Comment on lines +281 to +285
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where did you get the keys for priceMap?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we keep it static like previous???

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me see if I can find the correct keys


lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration)

// calculate region data transfer out to internet
dataTransferToInternet := (bytesUploaded / 1000000000) * 0.085
lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/10000000)

// calculate region data transfer out to origin
dataTransferToOrigin := (bytesDownloaded / 1000000000) * 0.02
functionInvocationsRate, _ := getRate(pricingOutput, "CloudFront Function Invocation", "function invocation")

// calculate requests cost
requestsCost := requests * 0.000001
functionInvocationsCost := functionInvocationsRate * functionInvocation

monthlyCost := dataTransferToInternet + dataTransferToOrigin + requestsCost
monthlyCost := dataTransferToInternetCost + dataTransferToOriginCost + requestsCost + lambdaEdgeDurationCost + lambdaEdgeRequestsCost + functionInvocationsCost

outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{
Resource: distribution.ARN,
Expand Down