Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Fix Cost Calculation for AWS CloudFront Distributions & Functions #1131

Closed
wants to merge 8 commits into from
Closed

feat: Fix Cost Calculation for AWS CloudFront Distributions & Functions #1131

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
265b209
fixed cost of cloudfront by distribution and function
bishal7679 Oct 23, 2023
ad9362d
modified pricemap key
bishal7679 Oct 24, 2023
fe0a48e
updated distribution with location function
bishal7679 Oct 29, 2023
4ddcb38
updated distribution with location based cost
bishal7679 Oct 30, 2023
8fcfcc7
updated cloudfront distribution cost based on locations
bishal7679 Oct 31, 2023
c4a576a
deleted println created for testing purpose for region
bishal7679 Oct 31, 2023
4832bb5
Merge branch 'develop' into feature/correct-cost-for-cloudfront
bishal7679 Oct 31, 2023
2dc22b6
Merge branch 'develop' into feature/correct-cost-for-cloudfront
bishal7679 Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/configuration/cloud-providers/aws.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ sidebar_label: Amazon Web Services
- API Gateway
- Access control lists
- CloudFront distributions
- CloudFront functions
- CloudWatch Dashboards
- CloudWatch alarms
- CloudWatch metrics
Expand Down
1 change: 1 addition & 0 deletions policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"apigateway:GET",
"cloudwatch:GetMetricStatistics",
"cloudfront:ListDistributions",
"cloudfront:Functions",
"cloudfront:ListTagsForResource",
"cloudwatch:DescribeAlarms",
"cloudwatch:ListTagsForResource",
Expand Down
1 change: 1 addition & 0 deletions providers/aws/aws.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ func listOfSupportedServices() []providers.FetchDataFunction {
ec2.Instances,
eks.KubernetesClusters,
cloudfront.Distributions,
cloudfront.Functions,
dynamodb.Tables,
ecs.Clusters,
ecs.TaskDefinitions,
Expand Down
61 changes: 50 additions & 11 deletions providers/aws/cloudfront/distributions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,19 @@ import (
"github.com/aws/aws-sdk-go-v2/service/cloudfront"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
"github.com/aws/aws-sdk-go-v2/service/pricing"
pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types"
. "github.com/tailwarden/komiser/models"
. "github.com/tailwarden/komiser/providers"
awsUtils "github.com/tailwarden/komiser/providers/aws/utils"
"github.com/tailwarden/komiser/utils"
)

const (
freeTierRequests = 10000000
freeTierUpload = 1099511627776
)

func Distributions(ctx context.Context, client ProviderClient) ([]Resource, error) {
resources := make([]Resource, 0)
var config cloudfront.ListDistributionsInput
Expand All @@ -25,6 +33,34 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
client.AWSClient.Region = "us-east-1"
cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient)
client.AWSClient.Region = tempRegion
pricingClient := pricing.NewFromConfig(*client.AWSClient)
bishal7679 marked this conversation as resolved.
Show resolved Hide resolved

pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{
ServiceCode: aws.String("AmazonCloudFront"),
Filters: []pricingTypes.Filter{
{
Field: aws.String("regionCode"),
Value: aws.String(client.AWSClient.Region),
Type: pricingTypes.FilterTypeTermMatch,
},
},
})
if err != nil {
log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err)
return resources, err
}

priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group")
if err != nil {
log.Errorf("ERROR: Failed to calculate cost per month: %v", err)
return resources, err
}

priceMapForRequest, err := awsUtils.GetPriceMap(pricingOutput, "requestType")
if err != nil {
log.Errorf("ERROR: Failed to calculate cost per month: %v", err)
return resources, err
}

for {
output, err := cloudfrontClient.ListDistributions(ctx, &config)
Expand All @@ -39,7 +75,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
MetricName: aws.String("BytesDownloaded"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -65,7 +101,7 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
MetricName: aws.String("BytesUploaded"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -84,14 +120,17 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
if metricsBytesUploadedOutput != nil && len(metricsBytesUploadedOutput.Datapoints) > 0 {
bytesUploaded = *metricsBytesUploadedOutput.Datapoints[0].Sum
}
if bytesUploaded > freeTierUpload {
bytesUploaded -= freeTierUpload
}

metricsRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Requests"),
Namespace: aws.String("AWS/CloudFront"),
Dimensions: []types.Dimension{
types.Dimension{
{
Name: aws.String("DistributionId"),
Value: distribution.Id,
},
Expand All @@ -110,17 +149,17 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
if metricsRequestsOutput != nil && len(metricsRequestsOutput.Datapoints) > 0 {
requests = *metricsRequestsOutput.Datapoints[0].Sum
}
if requests > freeTierRequests {
requests -= freeTierRequests
}

// calculate region data transfer out to internet
dataTransferToInternet := (bytesUploaded / 1000000000) * 0.085
dataTransferToInternetCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-In-Bytes"], (float64(bytesUploaded) / 1099511627776)*1024)

// calculate region data transfer out to origin
dataTransferToOrigin := (bytesDownloaded / 1000000000) * 0.02
dataTransferToOriginCost := awsUtils.GetCost(priceMap["AWS-CloudFront-DataTransfer-Out-Bytes"], (float64(bytesDownloaded) / 1099511627776)*1024)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a thought it would be better to declare a var for static values for better readability for others who might take ref of your work


// calculate requests cost
requestsCost := requests * 0.000001
requestsCost := awsUtils.GetCost(priceMapForRequest["CloudFront-Request-Origin-Shield"], requests/10000)

monthlyCost := dataTransferToInternet + dataTransferToOrigin + requestsCost
monthlyCost := dataTransferToInternetCost + dataTransferToOriginCost + requestsCost

outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{
Resource: distribution.ARN,
Expand Down Expand Up @@ -164,4 +203,4 @@ func Distributions(ctx context.Context, client ProviderClient) ([]Resource, erro
"resources": len(resources),
}).Info("Fetched resources")
return resources, nil
}
}
161 changes: 161 additions & 0 deletions providers/aws/cloudfront/functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package cloudfront

import (
"context"
"fmt"
"time"

log "github.com/sirupsen/logrus"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudfront"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
"github.com/aws/aws-sdk-go-v2/service/pricing"
pricingTypes "github.com/aws/aws-sdk-go-v2/service/pricing/types"
. "github.com/tailwarden/komiser/models"
. "github.com/tailwarden/komiser/providers"
awsUtils "github.com/tailwarden/komiser/providers/aws/utils"
"github.com/tailwarden/komiser/utils"
)

func Functions(ctx context.Context, client ProviderClient) ([]Resource, error) {
resources := make([]Resource, 0)
var config cloudfront.ListFunctionsInput
cloudfrontClient := cloudfront.NewFromConfig(*client.AWSClient)

tempRegion := client.AWSClient.Region
client.AWSClient.Region = "us-east-1"
cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient)
client.AWSClient.Region = tempRegion
pricingClient := pricing.NewFromConfig(*client.AWSClient)
bishal7679 marked this conversation as resolved.
Show resolved Hide resolved

pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{
ServiceCode: aws.String("AmazonCloudFront"),
Filters: []pricingTypes.Filter{
{
Field: aws.String("regionCode"),
Value: aws.String(client.AWSClient.Region),
Type: pricingTypes.FilterTypeTermMatch,
},
},
})
if err != nil {
log.Errorf("ERROR: Couldn't fetch pricing info for AWS CloudFront: %v", err)
return resources, err
bishal7679 marked this conversation as resolved.
Show resolved Hide resolved
}

priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group")
if err != nil {
log.Errorf("ERROR: Failed to calculate cost per month: %v", err)
return resources, err
}

for {
output, err := cloudfrontClient.ListFunctions(ctx, &config)
if err != nil {
return resources, err
}

for _, function := range output.FunctionList.Items {
metricsLambdaEdgeDurationOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Duration"),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does that work? If yes, that would be great 👍

Namespace: aws.String("AWS/LambdaEdge"),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that not aws.String("AWS/Cloudfront")?

Copy link
Contributor Author

@bishal7679 bishal7679 Oct 26, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Azanul WDYT??
AWS/Cloudfront or AWS/Lambda would be correct??

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AWS/CloudFront
ref: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/programming-cloudwatch-metrics.html#cloudfront-metrics-global-values

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

capital F

Dimensions: []types.Dimension{
{
Name: aws.String("FunctionName"),
Value: function.Name,
},
},
Period: aws.Int32(86400),
Statistics: []types.Statistic{
types.StatisticAverage,
},
})

if err != nil {
log.Warnf("Couldn't fetch Lambda@Edge Duration metric for %s", *function.Name)
}

lambdaEdgeDuration := 0.0
if metricsLambdaEdgeDurationOutput != nil && len(metricsLambdaEdgeDurationOutput.Datapoints) > 0 {
lambdaEdgeDuration = *metricsLambdaEdgeDurationOutput.Datapoints[0].Average
}

metricsLambdaEdgeRequestsOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{
StartTime: aws.Time(utils.BeginningOfMonth(time.Now())),
EndTime: aws.Time(time.Now()),
MetricName: aws.String("Requests"),
Namespace: aws.String("AWS/LambdaEdge"),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't it AWS/Cloudfront?

Dimensions: []types.Dimension{
{
Name: aws.String("FunctionName"),
Value: function.Name,
},
},
Period: aws.Int32(86400),
Statistics: []types.Statistic{
types.StatisticSum,
},
})

if err != nil {
log.Warnf("Couldn't fetch Lambda@Edge Requests metric for %s", *function.Name)
}

lambdaEdgeRequests := 0.0
if metricsLambdaEdgeRequestsOutput != nil && len(metricsLambdaEdgeRequestsOutput.Datapoints) > 0 {
lambdaEdgeRequests = *metricsLambdaEdgeRequestsOutput.Datapoints[0].Sum
}

lambdaEdgeDurationCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Duration"], lambdaEdgeDuration)

lambdaEdgeRequestsCost := awsUtils.GetCost(priceMap["AWS-Lambda-Edge-Requests"], lambdaEdgeRequests/10000000)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you should declare a constant for 10000000 this should be descriptive so other developers would have an idea of what this calculation achieves.


monthlyCost := lambdaEdgeDurationCost + lambdaEdgeRequestsCost

outputTags, err := cloudfrontClient.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{
Resource: function.FunctionMetadata.FunctionARN,
})

tags := make([]Tag, 0)

if err == nil {
for _, tag := range outputTags.Tags.Items {
tags = append(tags, Tag{
Key: *tag.Key,
Value: *tag.Value,
})
}
}

resources = append(resources, Resource{
Provider: "AWS",
Account: client.Name,
Service: "CloudFront",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't it be Cloudfront Functions?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cloudfront functions isn't a service, it's a resource/sub-resource of Cloudfront service

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right.

ResourceId: *function.FunctionMetadata.FunctionARN,
Region: client.AWSClient.Region,
Name: *function.Name,
Cost: monthlyCost,
Tags: tags,
FetchedAt: time.Now(),
Link: fmt.Sprintf("https://%s.console.aws.amazon.com/cloudfront/v3/home?region=%s#/distributions/%s", client.AWSClient.Region, client.AWSClient.Region, *function.Name),
})
}

if aws.ToString(output.FunctionList.NextMarker) == "" {
break
}
config.Marker = output.FunctionList.NextMarker
}
log.WithFields(log.Fields{
"provider": "AWS",
"account": client.Name,
"region": client.AWSClient.Region,
"service": "CloudFront",
Azanul marked this conversation as resolved.
Show resolved Hide resolved
"resources": len(resources),
}).Info("Fetched resources")
return resources, nil
}