From ce7d6680991fc299c6fe79740114cbdecaa6c00d Mon Sep 17 00:00:00 2001 From: horiodino Date: Sat, 28 Oct 2023 23:59:39 +0530 Subject: [PATCH 01/10] enhancement: added ssm based instance --- providers/aws/ec2/instances.go | 135 +++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index 32eb93c08..e67b4aa1a 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + cloudwatchTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" + "github.com/aws/aws-sdk-go-v2/service/ssm" + awsUtils "github.com/tailwarden/komiser/providers/aws/utils" "strconv" "time" @@ -18,6 +22,9 @@ import ( "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" "github.com/tailwarden/komiser/utils" + + . "github.com/tailwarden/komiser/models" + . "github.com/tailwarden/komiser/providers" ) func Instances(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { @@ -173,6 +180,134 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R return resources, nil } +// AIM : simple use the list of SSM managed instances to generate respective resources. You may fetch metric and calculate the cost +// in aws s3 bucket price is calculated per request so we need to calculate the cost per month +// but for ec2 it is calculated per hour so we need to calculate the cost per hour + +func ConvertBytesToTerabytes(bytes int64) float64 { + return float64(bytes) / 1099511627776 +} + +func getMangedEc2(ctx context.Context, client ProviderClient) ([]Resource, error) { + + resources := make([]Resource, 0) + var config = ssm.DescribeInstanceInformationInput{ + MaxResults: aws.Int32(100), + } + ssmClient := ssm.NewFromConfig(*client.AWSClient) + cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + pricingClient := pricing.NewFromConfig(*client.AWSClient) + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonEC2"), + Filters: []types.Filter{ + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: types.FilterTypeTermMatch, + }, + }, + }) + + if err != nil { + log.Errorf("ERROR: Couldn't fetch pricing info for AWS EC2: %v", err) + return resources, err + } + + priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") + if err != nil { + log.Errorf("ERROR: Failed to calculate cost per month: %v", err) + return resources, err + } + + output, err := ssmClient.DescribeInstanceInformation(ctx, &config) + if err != nil { + return nil, err + } + + for _, ec2 := range output.InstanceInformationList { + metricesEc2sizebyOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("EC2SizeBytes"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []cloudwatchTypes.Dimension{ + { + Name: aws.String("InstanceName"), + Value: ec2.Name, + }, + }, + Unit: cloudwatchTypes.StandardUnitBytes, + Period: aws.Int32(3600), + Statistics: []cloudwatchTypes.Statistic{ + cloudwatchTypes.StatisticAverage, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", *bucket.Name) + } + + instanceType := "" + if ec2.InstanceType != nil { + instanceType = *ec2.InstanceType + } + + + + sizeInTB := 0.0 + if ec2.Ins + + metricesUsesOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ + StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), + EndTime: aws.Time(time.Now()), + MetricName: aws.String("AllRequests"), + Namespace: aws.String("AWS/EC2"), + Dimensions: []cloudwatchTypes.Dimension{ + { + Name: aws.String("InstanceName"), + Value: ec2.Name, + }, + }, + Unit: cloudwatchTypes.StandardUnitCount, + Period: aws.Int32(3600), + Statistics: []cloudwatchTypes.Statistic{ + cloudwatchTypes.StatisticAverage, + }, + }) + + if err != nil { + log.Warnf("Couldn't fetch usage metric for %s", *bucket.Name) + } + + + + sizeCharges := 0.0 + if metricesUsesOutput != nil && len(metricesUsesOutput.Datapoints) > 0 { + sizeCharges = *metricesUsesOutput.Datapoints[0].Average + } + + monthlyCost := sizeCharges * priceMap[*ec2.PlatformName] + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "EC2", + Region: client.AWSClient.Region, + ResourceId: *ec2.InstanceId, + Name: *ec2.Name, + Cost: monthlyCost, + CreatedAt: *ec2.RegistrationDate, + Tags: tags + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2.InstanceId), + }) + + } + + return nil, nil +} + func getEC2Relations(inst *etype.Instance, resourceArn string) (rel []models.Link) { // Get associated security groups for _, sgrp := range inst.SecurityGroups { From a4d260ddd4791f70bef2635b38b6137b00fdb9b1 Mon Sep 17 00:00:00 2001 From: horiodino Date: Sun, 29 Oct 2023 12:40:44 +0530 Subject: [PATCH 02/10] enhancement: added cost-monitoring on smm based instances --- providers/aws/ec2/instances.go | 258 +++++++++++++++++++++------------ 1 file changed, 163 insertions(+), 95 deletions(-) diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index e67b4aa1a..48cf32be8 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -4,13 +4,10 @@ import ( "context" "encoding/json" "fmt" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - cloudwatchTypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/aws/aws-sdk-go-v2/service/ssm" - awsUtils "github.com/tailwarden/komiser/providers/aws/utils" "strconv" "time" + "github.com/aws/aws-sdk-go-v2/service/ssm" log "github.com/sirupsen/logrus" "github.com/aws/aws-sdk-go-v2/aws" @@ -184,10 +181,6 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R // in aws s3 bucket price is calculated per request so we need to calculate the cost per month // but for ec2 it is calculated per hour so we need to calculate the cost per hour -func ConvertBytesToTerabytes(bytes int64) float64 { - return float64(bytes) / 1099511627776 -} - func getMangedEc2(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) @@ -195,117 +188,192 @@ func getMangedEc2(ctx context.Context, client ProviderClient) ([]Resource, error MaxResults: aws.Int32(100), } ssmClient := ssm.NewFromConfig(*client.AWSClient) - cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + //cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) pricingClient := pricing.NewFromConfig(*client.AWSClient) - pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ - ServiceCode: aws.String("AmazonEC2"), - Filters: []types.Filter{ - { - Field: aws.String("regionCode"), - Value: aws.String(client.AWSClient.Region), - Type: types.FilterTypeTermMatch, - }, - }, - }) - - if err != nil { - log.Errorf("ERROR: Couldn't fetch pricing info for AWS EC2: %v", err) - return resources, err - } - - priceMap, err := awsUtils.GetPriceMap(pricingOutput, "group") - if err != nil { - log.Errorf("ERROR: Failed to calculate cost per month: %v", err) - return resources, err - } - output, err := ssmClient.DescribeInstanceInformation(ctx, &config) if err != nil { return nil, err } - for _, ec2 := range output.InstanceInformationList { - metricesEc2sizebyOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("EC2SizeBytes"), - Namespace: aws.String("AWS/EC2"), - Dimensions: []cloudwatchTypes.Dimension{ - { - Name: aws.String("InstanceName"), - Value: ec2.Name, - }, - }, - Unit: cloudwatchTypes.StandardUnitBytes, - Period: aws.Int32(3600), - Statistics: []cloudwatchTypes.Statistic{ - cloudwatchTypes.StatisticAverage, - }, - }) - + for _, ec2instance := range output.InstanceInformationList { + running, err := isRunning(ctx, *ec2instance.InstanceId, client) if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", *bucket.Name) + return nil, err } - instanceType := "" - if ec2.InstanceType != nil { - instanceType = *ec2.InstanceType - } + if running { + startOfMonth := utils.BeginningOfMonth(time.Now()) + hourlyUsage := 0 + hourlyUsage, err := getHourlyUses(ctx, *ec2instance.InstanceId, client, startOfMonth) + if err != nil { + return nil, err + } - sizeInTB := 0.0 - if ec2.Ins + instancetype, err := getInstanceType(ctx, *ec2instance.InstanceId, client) + if err != nil { + return nil, err + } - metricesUsesOutput, err := cloudwatchClient.GetMetricStatistics(ctx, &cloudwatch.GetMetricStatisticsInput{ - StartTime: aws.Time(utils.BeginningOfMonth(time.Now())), - EndTime: aws.Time(time.Now()), - MetricName: aws.String("AllRequests"), - Namespace: aws.String("AWS/EC2"), - Dimensions: []cloudwatchTypes.Dimension{ - { - Name: aws.String("InstanceName"), - Value: ec2.Name, + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonEC2"), + Filters: []types.Filter{ + { + Field: aws.String("operatingSystem"), + Value: aws.String("linux"), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("instanceType"), + Value: aws.String(instancetype), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("capacitystatus"), + Value: aws.String("Used"), + Type: types.FilterTypeTermMatch, + }, }, - }, - Unit: cloudwatchTypes.StandardUnitCount, - Period: aws.Int32(3600), - Statistics: []cloudwatchTypes.Statistic{ - cloudwatchTypes.StatisticAverage, - }, - }) + MaxResults: aws.Int32(1), + }) + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) + } - if err != nil { - log.Warnf("Couldn't fetch usage metric for %s", *bucket.Name) + log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) + + hourlyCost := 0.0 + montlyCost := 0.0 + + if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { + + pricingResult := models.PricingResult{} + err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) + if err != nil { + log.Fatalf("Failed to unmarshal JSON: %v", err) + } + + for _, onDemand := range pricingResult.Terms.OnDemand { + for _, priceDimension := range onDemand.PriceDimensions { + hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) + if err != nil { + log.Fatalf("Failed to parse hourly cost: %v", err) + } + break + } + break + } + } + + montlyCost = float64(hourlyUsage) * hourlyCost + + tagsResp, err := ssmClient.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ + ResourceId: ec2instance.InstanceId, + }) + + tags := make([]Tag, 0) + if err == nil { + for _, t := range tagsResp.TagList { + tags = append(tags, Tag{ + Key: *t.Key, + Value: *t.Value, + }) + } + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "EC2", + Region: client.AWSClient.Region, + ResourceId: *ec2instance.InstanceId, + Name: *ec2instance.Name, + Cost: montlyCost, + CreatedAt: *ec2instance.RegistrationDate, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), + }) + } + + } + + return resources, nil +} + +func getInstanceType(ctx context.Context, instanceId string, client ProviderClient) (instanceType string, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceId}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) + + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return "", err + } + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + instanceType := string(instance.InstanceType) + return instanceType, nil } + } + return "", nil +} - +func isRunning(ctx context.Context, instanceId string, client ProviderClient) (running bool, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceId}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) - sizeCharges := 0.0 - if metricesUsesOutput != nil && len(metricesUsesOutput.Datapoints) > 0 { - sizeCharges = *metricesUsesOutput.Datapoints[0].Average + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return false, err + } + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + if instance.State.Name != "stopped" { + return true, nil + } } + } + return false, nil +} - monthlyCost := sizeCharges * priceMap[*ec2.PlatformName] - - resources = append(resources, Resource{ - Provider: "AWS", - Account: client.Name, - Service: "EC2", - Region: client.AWSClient.Region, - ResourceId: *ec2.InstanceId, - Name: *ec2.Name, - Cost: monthlyCost, - CreatedAt: *ec2.RegistrationDate, - Tags: tags - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2.InstanceId), - }) +func getHourlyUses(ctx context.Context, instanceID string, client ProviderClient, startOfMonth time.Time) (hourlyUsage int, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return 0, err + } + + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + if instance.LaunchTime.Before(startOfMonth) { + hourlyUsage = int(time.Since(startOfMonth).Hours()) + return hourlyUsage, nil + } else { + hourlyUsage = int(time.Since(*instance.LaunchTime).Hours()) + return hourlyUsage, nil + } + } } - return nil, nil + return 0, nil } func getEC2Relations(inst *etype.Instance, resourceArn string) (rel []models.Link) { From b9926c916fb72d87de0fecfd3fc050c5f6cdfccb Mon Sep 17 00:00:00 2001 From: horiodino Date: Wed, 15 Nov 2023 10:11:26 +0530 Subject: [PATCH 03/10] Refactor: Removed dot imports and organized changes into the systemsmanager folder. --- providers/aws/ec2/instances.go | 203 ---------------- .../aws/systemsmanager/managedinstances.go | 219 ++++++++++++++++++ 2 files changed, 219 insertions(+), 203 deletions(-) create mode 100644 providers/aws/systemsmanager/managedinstances.go diff --git a/providers/aws/ec2/instances.go b/providers/aws/ec2/instances.go index 48cf32be8..32eb93c08 100644 --- a/providers/aws/ec2/instances.go +++ b/providers/aws/ec2/instances.go @@ -7,7 +7,6 @@ import ( "strconv" "time" - "github.com/aws/aws-sdk-go-v2/service/ssm" log "github.com/sirupsen/logrus" "github.com/aws/aws-sdk-go-v2/aws" @@ -19,9 +18,6 @@ import ( "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" "github.com/tailwarden/komiser/utils" - - . "github.com/tailwarden/komiser/models" - . "github.com/tailwarden/komiser/providers" ) func Instances(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { @@ -177,205 +173,6 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R return resources, nil } -// AIM : simple use the list of SSM managed instances to generate respective resources. You may fetch metric and calculate the cost -// in aws s3 bucket price is calculated per request so we need to calculate the cost per month -// but for ec2 it is calculated per hour so we need to calculate the cost per hour - -func getMangedEc2(ctx context.Context, client ProviderClient) ([]Resource, error) { - - resources := make([]Resource, 0) - var config = ssm.DescribeInstanceInformationInput{ - MaxResults: aws.Int32(100), - } - ssmClient := ssm.NewFromConfig(*client.AWSClient) - //cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) - pricingClient := pricing.NewFromConfig(*client.AWSClient) - - output, err := ssmClient.DescribeInstanceInformation(ctx, &config) - if err != nil { - return nil, err - } - - for _, ec2instance := range output.InstanceInformationList { - running, err := isRunning(ctx, *ec2instance.InstanceId, client) - if err != nil { - return nil, err - } - - if running { - - startOfMonth := utils.BeginningOfMonth(time.Now()) - hourlyUsage := 0 - - hourlyUsage, err := getHourlyUses(ctx, *ec2instance.InstanceId, client, startOfMonth) - if err != nil { - return nil, err - } - - instancetype, err := getInstanceType(ctx, *ec2instance.InstanceId, client) - if err != nil { - return nil, err - } - - pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ - ServiceCode: aws.String("AmazonEC2"), - Filters: []types.Filter{ - { - Field: aws.String("operatingSystem"), - Value: aws.String("linux"), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("instanceType"), - Value: aws.String(instancetype), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("regionCode"), - Value: aws.String(client.AWSClient.Region), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("capacitystatus"), - Value: aws.String("Used"), - Type: types.FilterTypeTermMatch, - }, - }, - MaxResults: aws.Int32(1), - }) - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) - } - - log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) - - hourlyCost := 0.0 - montlyCost := 0.0 - - if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { - - pricingResult := models.PricingResult{} - err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) - if err != nil { - log.Fatalf("Failed to unmarshal JSON: %v", err) - } - - for _, onDemand := range pricingResult.Terms.OnDemand { - for _, priceDimension := range onDemand.PriceDimensions { - hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) - if err != nil { - log.Fatalf("Failed to parse hourly cost: %v", err) - } - break - } - break - } - } - - montlyCost = float64(hourlyUsage) * hourlyCost - - tagsResp, err := ssmClient.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ - ResourceId: ec2instance.InstanceId, - }) - - tags := make([]Tag, 0) - if err == nil { - for _, t := range tagsResp.TagList { - tags = append(tags, Tag{ - Key: *t.Key, - Value: *t.Value, - }) - } - } - - resources = append(resources, Resource{ - Provider: "AWS", - Account: client.Name, - Service: "EC2", - Region: client.AWSClient.Region, - ResourceId: *ec2instance.InstanceId, - Name: *ec2instance.Name, - Cost: montlyCost, - CreatedAt: *ec2instance.RegistrationDate, - Tags: tags, - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), - }) - } - - } - - return resources, nil -} - -func getInstanceType(ctx context.Context, instanceId string, client ProviderClient) (instanceType string, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceId}, - MaxResults: aws.Int32(1), - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return "", err - } - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - instanceType := string(instance.InstanceType) - return instanceType, nil - } - } - return "", nil -} - -func isRunning(ctx context.Context, instanceId string, client ProviderClient) (running bool, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceId}, - MaxResults: aws.Int32(1), - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return false, err - } - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - if instance.State.Name != "stopped" { - return true, nil - } - } - } - return false, nil -} - -func getHourlyUses(ctx context.Context, instanceID string, client ProviderClient, startOfMonth time.Time) (hourlyUsage int, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceID}, - MaxResults: aws.Int32(1), - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return 0, err - } - - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - if instance.LaunchTime.Before(startOfMonth) { - hourlyUsage = int(time.Since(startOfMonth).Hours()) - return hourlyUsage, nil - } else { - hourlyUsage = int(time.Since(*instance.LaunchTime).Hours()) - return hourlyUsage, nil - } - } - } - - return 0, nil -} - func getEC2Relations(inst *etype.Instance, resourceArn string) (rel []models.Link) { // Get associated security groups for _, sgrp := range inst.SecurityGroups { diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go new file mode 100644 index 000000000..ba406b1dd --- /dev/null +++ b/providers/aws/systemsmanager/managedinstances.go @@ -0,0 +1,219 @@ +package systemsmanager + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ssm" + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/pricing" + "github.com/aws/aws-sdk-go-v2/service/pricing/types" + "github.com/tailwarden/komiser/models" + "github.com/tailwarden/komiser/providers" + "github.com/tailwarden/komiser/utils" +) + +// AIM : simple use the list of SSM managed instances to generate respective resources. You may fetch metric and calculate the cost +// in aws s3 bucket price is calculated per request so we need to calculate the cost per month +// but for ec2 it is calculated per hour so we need to calculate the cost per hour + +func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) + + resources := make([]Resource, 0) + var config = ssm.DescribeInstanceInformationInput{ + MaxResults: aws.Int32(100), + } + ssmClient := ssm.NewFromConfig(*client.AWSClient) + //cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) + pricingClient := pricing.NewFromConfig(*client.AWSClient) + + output, err := ssmClient.DescribeInstanceInformation(ctx, &config) + if err != nil { + return nil, err + } + + for _, ec2instance := range output.InstanceInformationList { + running, err := isRunning(ctx, *ec2instance.InstanceId, client) + if err != nil { + return nil, err + } + + if running { + + startOfMonth := utils.BeginningOfMonth(time.Now()) + hourlyUsage := 0 + + hourlyUsage, err := getHourlyUses(ctx, *ec2instance.InstanceId, client, startOfMonth) + if err != nil { + return nil, err + } + + instancetype, err := getInstanceType(ctx, *ec2instance.InstanceId, client) + if err != nil { + return nil, err + } + + pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ + ServiceCode: aws.String("AmazonEC2"), + Filters: []types.Filter{ + { + Field: aws.String("operatingSystem"), + Value: aws.String("linux"), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("instanceType"), + Value: aws.String(instancetype), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("regionCode"), + Value: aws.String(client.AWSClient.Region), + Type: types.FilterTypeTermMatch, + }, + { + Field: aws.String("capacitystatus"), + Value: aws.String("Used"), + Type: types.FilterTypeTermMatch, + }, + }, + MaxResults: aws.Int32(1), + }) + if err != nil { + log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) + } + + log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) + + hourlyCost := 0.0 + montlyCost := 0.0 + + if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { + + pricingResult := models.PricingResult{} + err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) + if err != nil { + log.Fatalf("Failed to unmarshal JSON: %v", err) + } + + for _, onDemand := range pricingResult.Terms.OnDemand { + for _, priceDimension := range onDemand.PriceDimensions { + hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) + if err != nil { + log.Fatalf("Failed to parse hourly cost: %v", err) + } + break + } + break + } + } + + montlyCost = float64(hourlyUsage) * hourlyCost + + tagsResp, err := ssmClient.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ + ResourceId: ec2instance.InstanceId, + }) + + tags := make([]Tag, 0) + if err == nil { + for _, t := range tagsResp.TagList { + tags = append(tags, Tag{ + Key: *t.Key, + Value: *t.Value, + }) + } + } + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "EC2", + Region: client.AWSClient.Region, + ResourceId: *ec2instance.InstanceId, + Name: *ec2instance.Name, + Cost: montlyCost, + CreatedAt: *ec2instance.RegistrationDate, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), + }) + } + + } + + return resources, nil +} + +func getInstanceType(ctx context.Context, instanceId string, client ProviderClient) (instanceType string, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceId}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) + + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return "", err + } + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + instanceType := string(instance.InstanceType) + return instanceType, nil + } + } + return "", nil +} + +func isRunning(ctx context.Context, instanceId string, client ProviderClient) (running bool, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceId}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) + + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return false, err + } + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + if instance.State.Name != "stopped" { + return true, nil + } + } + } + return false, nil +} + +func getHourlyUses(ctx context.Context, instanceID string, client ProviderClient, startOfMonth time.Time) (hourlyUsage int, err error) { + var config = ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + MaxResults: aws.Int32(1), + } + ec2Client := ec2.NewFromConfig(*client.AWSClient) + + output, err := ec2Client.DescribeInstances(ctx, &config) + if err != nil { + return 0, err + } + + for _, reservations := range output.Reservations { + for _, instance := range reservations.Instances { + if instance.LaunchTime.Before(startOfMonth) { + hourlyUsage = int(time.Since(startOfMonth).Hours()) + return hourlyUsage, nil + } else { + hourlyUsage = int(time.Since(*instance.LaunchTime).Hours()) + return hourlyUsage, nil + } + } + } + + return 0, nil +} From bfaa0efe1b1bc04d372d7c53d49766d74e5cdab7 Mon Sep 17 00:00:00 2001 From: horiodino Date: Fri, 17 Nov 2023 20:27:10 +0530 Subject: [PATCH 04/10] enhancement: Removed the cost calculation logic and related code --- .../aws/systemsmanager/managedinstances.go | 162 +++--------------- 1 file changed, 21 insertions(+), 141 deletions(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index ba406b1dd..6d2fcd613 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -2,38 +2,24 @@ package systemsmanager import ( "context" - "encoding/json" "fmt" - "strconv" - "time" - - "github.com/aws/aws-sdk-go-v2/service/ssm" - log "github.com/sirupsen/logrus" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/pricing" - "github.com/aws/aws-sdk-go-v2/service/pricing/types" + "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" - "github.com/tailwarden/komiser/utils" ) -// AIM : simple use the list of SSM managed instances to generate respective resources. You may fetch metric and calculate the cost -// in aws s3 bucket price is calculated per request so we need to calculate the cost per month -// but for ec2 it is calculated per hour so we need to calculate the cost per hour +func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { + resources := make([]models.Resource, 0) -func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) + ssmClient := ssm.NewFromConfig(*client.AWSClient) - resources := make([]Resource, 0) - var config = ssm.DescribeInstanceInformationInput{ + output, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ MaxResults: aws.Int32(100), - } - ssmClient := ssm.NewFromConfig(*client.AWSClient) - //cloudwatchClient := cloudwatch.NewFromConfig(*client.AWSClient) - pricingClient := pricing.NewFromConfig(*client.AWSClient) + }) - output, err := ssmClient.DescribeInstanceInformation(ctx, &config) if err != nil { return nil, err } @@ -45,114 +31,33 @@ func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]model } if running { - - startOfMonth := utils.BeginningOfMonth(time.Now()) - hourlyUsage := 0 - - hourlyUsage, err := getHourlyUses(ctx, *ec2instance.InstanceId, client, startOfMonth) - if err != nil { - return nil, err - } - - instancetype, err := getInstanceType(ctx, *ec2instance.InstanceId, client) + instanceType, err := getInstanceType(ctx, *ec2instance.InstanceId, client) if err != nil { return nil, err } - pricingOutput, err := pricingClient.GetProducts(ctx, &pricing.GetProductsInput{ - ServiceCode: aws.String("AmazonEC2"), - Filters: []types.Filter{ - { - Field: aws.String("operatingSystem"), - Value: aws.String("linux"), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("instanceType"), - Value: aws.String(instancetype), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("regionCode"), - Value: aws.String(client.AWSClient.Region), - Type: types.FilterTypeTermMatch, - }, - { - Field: aws.String("capacitystatus"), - Value: aws.String("Used"), - Type: types.FilterTypeTermMatch, - }, - }, - MaxResults: aws.Int32(1), - }) - if err != nil { - log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) - } - - log.Warnf("Couldn't fetch invocations metric for %s", ec2instance.Name) - - hourlyCost := 0.0 - montlyCost := 0.0 - - if pricingOutput != nil && len(pricingOutput.PriceList) > 0 { - - pricingResult := models.PricingResult{} - err := json.Unmarshal([]byte(pricingOutput.PriceList[0]), &pricingResult) - if err != nil { - log.Fatalf("Failed to unmarshal JSON: %v", err) - } - - for _, onDemand := range pricingResult.Terms.OnDemand { - for _, priceDimension := range onDemand.PriceDimensions { - hourlyCost, err = strconv.ParseFloat(priceDimension.PricePerUnit.USD, 64) - if err != nil { - log.Fatalf("Failed to parse hourly cost: %v", err) - } - break - } - break - } - } - - montlyCost = float64(hourlyUsage) * hourlyCost - - tagsResp, err := ssmClient.ListTagsForResource(ctx, &ssm.ListTagsForResourceInput{ - ResourceId: ec2instance.InstanceId, - }) - - tags := make([]Tag, 0) - if err == nil { - for _, t := range tagsResp.TagList { - tags = append(tags, Tag{ - Key: *t.Key, - Value: *t.Value, - }) - } - } - - resources = append(resources, Resource{ + resources = append(resources, models.Resource{ Provider: "AWS", Account: client.Name, Service: "EC2", Region: client.AWSClient.Region, ResourceId: *ec2instance.InstanceId, Name: *ec2instance.Name, - Cost: montlyCost, + Cost: 0.0, // No cost calculation in this version CreatedAt: *ec2instance.RegistrationDate, - Tags: tags, - FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), + Tags: nil, // No tags in this version + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", + client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), }) } - } return resources, nil } -func getInstanceType(ctx context.Context, instanceId string, client ProviderClient) (instanceType string, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceId}, +func getInstanceType(ctx context.Context, instanceID string, client providers.ProviderClient) (instanceType string, err error) { + config := ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, MaxResults: aws.Int32(1), } ec2Client := ec2.NewFromConfig(*client.AWSClient) @@ -161,6 +66,7 @@ func getInstanceType(ctx context.Context, instanceId string, client ProviderClie if err != nil { return "", err } + for _, reservations := range output.Reservations { for _, instance := range reservations.Instances { instanceType := string(instance.InstanceType) @@ -170,9 +76,9 @@ func getInstanceType(ctx context.Context, instanceId string, client ProviderClie return "", nil } -func isRunning(ctx context.Context, instanceId string, client ProviderClient) (running bool, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceId}, +func isRunning(ctx context.Context, instanceID string, client providers.ProviderClient) (running bool, err error) { + config := ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, MaxResults: aws.Int32(1), } ec2Client := ec2.NewFromConfig(*client.AWSClient) @@ -181,39 +87,13 @@ func isRunning(ctx context.Context, instanceId string, client ProviderClient) (r if err != nil { return false, err } + for _, reservations := range output.Reservations { for _, instance := range reservations.Instances { - if instance.State.Name != "stopped" { + if instance.State.Name != ec2.InstanceStateNameStopped { return true, nil } } } return false, nil } - -func getHourlyUses(ctx context.Context, instanceID string, client ProviderClient, startOfMonth time.Time) (hourlyUsage int, err error) { - var config = ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceID}, - MaxResults: aws.Int32(1), - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return 0, err - } - - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - if instance.LaunchTime.Before(startOfMonth) { - hourlyUsage = int(time.Since(startOfMonth).Hours()) - return hourlyUsage, nil - } else { - hourlyUsage = int(time.Since(*instance.LaunchTime).Hours()) - return hourlyUsage, nil - } - } - } - - return 0, nil -} From 304128dbaaa983edf0051c2c662fcc49f6ed20b2 Mon Sep 17 00:00:00 2001 From: horiodino Date: Fri, 24 Nov 2023 12:18:13 +0530 Subject: [PATCH 05/10] implemented suggested changes --- .../aws/systemsmanager/managedinstances.go | 88 ++++++++++++------- 1 file changed, 57 insertions(+), 31 deletions(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index 6d2fcd613..80f83d6fe 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -3,10 +3,14 @@ package systemsmanager import ( "context" "fmt" + "log" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/tailwarden/komiser/models" "github.com/tailwarden/komiser/providers" ) @@ -17,69 +21,76 @@ func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]model ssmClient := ssm.NewFromConfig(*client.AWSClient) output, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ - MaxResults: aws.Int32(100), + MaxResults: aws.Int32(50), }) - if err != nil { - return nil, err + log.Fatal(err) } for _, ec2instance := range output.InstanceInformationList { running, err := isRunning(ctx, *ec2instance.InstanceId, client) if err != nil { - return nil, err + log.Fatal(err) } - if running { - instanceType, err := getInstanceType(ctx, *ec2instance.InstanceId, client) + if true == running { + fmt.Println("running") + instance, err := getInstanceType(ctx, *ec2instance.InstanceId, client) if err != nil { return nil, err } - resources = append(resources, models.Resource{ - Provider: "AWS", - Account: client.Name, - Service: "EC2", - Region: client.AWSClient.Region, - ResourceId: *ec2instance.InstanceId, - Name: *ec2instance.Name, - Cost: 0.0, // No cost calculation in this version - CreatedAt: *ec2instance.RegistrationDate, - Tags: nil, // No tags in this version - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", - client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), - }) + if len(instance.Reservations) > 0 && len(instance.Reservations[0].Instances) > 0 { + currentInstance := instance.Reservations[0].Instances[0] + + tags := make([]models.Tag, 0) + for _, tag := range currentInstance.Tags { + tags = append(tags, models.Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + account, account_id := fetchid() + + resources = append(resources, models.Resource{ + Provider: "AWS", + Account: account, + AccountId: account_id, + Service: "EC2", + Region: client.AWSClient.Region, + ResourceId: *currentInstance.InstanceId, + Name: string(currentInstance.InstanceType), + CreatedAt: *currentInstance.LaunchTime, + FetchedAt: time.Now(), + Tags: tags, + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", + client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), + }) + } } } return resources, nil } -func getInstanceType(ctx context.Context, instanceID string, client providers.ProviderClient) (instanceType string, err error) { +func getInstanceType(ctx context.Context, instanceID string, client providers.ProviderClient) (*ec2.DescribeInstancesOutput, error) { config := ec2.DescribeInstancesInput{ InstanceIds: []string{instanceID}, - MaxResults: aws.Int32(1), } ec2Client := ec2.NewFromConfig(*client.AWSClient) output, err := ec2Client.DescribeInstances(ctx, &config) if err != nil { - return "", err + return output, err } - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - instanceType := string(instance.InstanceType) - return instanceType, nil - } - } - return "", nil + return output, nil } func isRunning(ctx context.Context, instanceID string, client providers.ProviderClient) (running bool, err error) { + fmt.Println("status check for instance id: ", instanceID) config := ec2.DescribeInstancesInput{ InstanceIds: []string{instanceID}, - MaxResults: aws.Int32(1), } ec2Client := ec2.NewFromConfig(*client.AWSClient) @@ -90,10 +101,25 @@ func isRunning(ctx context.Context, instanceID string, client providers.Provider for _, reservations := range output.Reservations { for _, instance := range reservations.Instances { - if instance.State.Name != ec2.InstanceStateNameStopped { + if instance.State.Name == types.InstanceStateNameRunning { return true, nil } } } return false, nil } + +func fetchid() (accountID string, userid string) { + svc := sts.NewFromConfig(*newclient()) + result, err := svc.GetCallerIdentity(context.Background(), &sts.GetCallerIdentityInput{}) + if err != nil { + fmt.Println("Got error retrieving account ID:") + log.Println(err.Error()) + } + + // if the accountID and userid same then the account is root account + accountID = *result.Account + userid = *result.UserId + + return accountID, userid +} From 59189f785aa63b04f595a7b3b549b409a1a8e87c Mon Sep 17 00:00:00 2001 From: horiodino Date: Fri, 8 Dec 2023 23:25:31 +0530 Subject: [PATCH 06/10] Reduced API Requests & bulk data fetching for instances. --- .../aws/systemsmanager/managedinstances.go | 100 ++++++------------ 1 file changed, 33 insertions(+), 67 deletions(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index 80f83d6fe..45ff00df2 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -15,111 +15,77 @@ import ( "github.com/tailwarden/komiser/providers" ) -func getMangedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { +func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { resources := make([]models.Resource, 0) ssmClient := ssm.NewFromConfig(*client.AWSClient) + ec2Client := ec2.NewFromConfig(*client.AWSClient) - output, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ + ssmOutput, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ MaxResults: aws.Int32(50), }) if err != nil { log.Fatal(err) } - for _, ec2instance := range output.InstanceInformationList { - running, err := isRunning(ctx, *ec2instance.InstanceId, client) - if err != nil { - log.Fatal(err) - } + instanceIds := make([]string, 0, len(ssmOutput.InstanceInformationList)) + for _, ec2instance := range ssmOutput.InstanceInformationList { + instanceIds = append(instanceIds, *ec2instance.InstanceId) + } + ec2Output, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: instanceIds, + }) + if err != nil { + return nil, err + } - if true == running { - fmt.Println("running") - instance, err := getInstanceType(ctx, *ec2instance.InstanceId, client) - if err != nil { - return nil, err - } + account, accountID, err := fetchID(ctx, client) + if err != nil { + return nil, err + } - if len(instance.Reservations) > 0 && len(instance.Reservations[0].Instances) > 0 { - currentInstance := instance.Reservations[0].Instances[0] + for _, ec2instance := range ec2Output.Reservations { + for _, instance := range ec2instance.Instances { + if instance.State.Name == types.InstanceStateNameRunning { tags := make([]models.Tag, 0) - for _, tag := range currentInstance.Tags { + for _, tag := range instance.Tags { tags = append(tags, models.Tag{ Key: *tag.Key, Value: *tag.Value, }) } - account, account_id := fetchid() resources = append(resources, models.Resource{ Provider: "AWS", Account: account, - AccountId: account_id, + AccountId: accountID, Service: "EC2", Region: client.AWSClient.Region, - ResourceId: *currentInstance.InstanceId, - Name: string(currentInstance.InstanceType), - CreatedAt: *currentInstance.LaunchTime, + ResourceId: *instance.InstanceId, + Name: string(instance.InstanceType), + CreatedAt: *instance.LaunchTime, FetchedAt: time.Now(), Tags: tags, Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", - client.AWSClient.Region, client.AWSClient.Region, *ec2instance.InstanceId), + client.AWSClient.Region, client.AWSClient.Region, *instance.InstanceId), }) } + } } return resources, nil } -func getInstanceType(ctx context.Context, instanceID string, client providers.ProviderClient) (*ec2.DescribeInstancesOutput, error) { - config := ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceID}, - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return output, err - } - - return output, nil -} - -func isRunning(ctx context.Context, instanceID string, client providers.ProviderClient) (running bool, err error) { - fmt.Println("status check for instance id: ", instanceID) - config := ec2.DescribeInstancesInput{ - InstanceIds: []string{instanceID}, - } - ec2Client := ec2.NewFromConfig(*client.AWSClient) - - output, err := ec2Client.DescribeInstances(ctx, &config) - if err != nil { - return false, err - } - - for _, reservations := range output.Reservations { - for _, instance := range reservations.Instances { - if instance.State.Name == types.InstanceStateNameRunning { - return true, nil - } - } - } - return false, nil -} - -func fetchid() (accountID string, userid string) { - svc := sts.NewFromConfig(*newclient()) - result, err := svc.GetCallerIdentity(context.Background(), &sts.GetCallerIdentityInput{}) +func fetchID(ctx context.Context, client providers.ProviderClient) (accountID string, userID string, err error) { + svc := sts.NewFromConfig(*client.AWSClient) + result, err := svc.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) if err != nil { fmt.Println("Got error retrieving account ID:") - log.Println(err.Error()) + return "", "", err } - // if the accountID and userid same then the account is root account - accountID = *result.Account - userid = *result.UserId - - return accountID, userid + // if the accountID and userID are the same, then the account is a root account + return *result.Account, *result.UserId, nil } From 18ee21d7a26c011d1f1d6ab0db84e86eb7783ae0 Mon Sep 17 00:00:00 2001 From: Praful Khanduri <99384392+Horiodino@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:54:04 +0000 Subject: [PATCH 07/10] added suggested changes --- providers/aws/systemsmanager/managedinstances.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index 45ff00df2..b7d14e69a 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -25,7 +25,7 @@ func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]mode MaxResults: aws.Int32(50), }) if err != nil { - log.Fatal(err) + return resources, err } instanceIds := make([]string, 0, len(ssmOutput.InstanceInformationList)) @@ -36,12 +36,12 @@ func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]mode InstanceIds: instanceIds, }) if err != nil { - return nil, err + return resources, err } account, accountID, err := fetchID(ctx, client) if err != nil { - return nil, err + return resources, err } for _, ec2instance := range ec2Output.Reservations { @@ -60,7 +60,7 @@ func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]mode Provider: "AWS", Account: account, AccountId: accountID, - Service: "EC2", + Service: "SSM-Instance", Region: client.AWSClient.Region, ResourceId: *instance.InstanceId, Name: string(instance.InstanceType), From 6002acbe4e9b75201198d94a66542fe1a172667d Mon Sep 17 00:00:00 2001 From: Praful Khanduri <99384392+Horiodino@users.noreply.github.com> Date: Sun, 7 Jan 2024 23:08:44 +0530 Subject: [PATCH 08/10] added pagination token for fetching ssm-instance-info --- .../aws/systemsmanager/managedinstances.go | 96 ++++++++++--------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index b7d14e69a..41e96e84f 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -3,10 +3,8 @@ package systemsmanager import ( "context" "fmt" - "log" "time" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -21,58 +19,66 @@ func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]mode ssmClient := ssm.NewFromConfig(*client.AWSClient) ec2Client := ec2.NewFromConfig(*client.AWSClient) - ssmOutput, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ - MaxResults: aws.Int32(50), - }) - if err != nil { - return resources, err - } + var nexttoken *string - instanceIds := make([]string, 0, len(ssmOutput.InstanceInformationList)) - for _, ec2instance := range ssmOutput.InstanceInformationList { - instanceIds = append(instanceIds, *ec2instance.InstanceId) - } - ec2Output, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - InstanceIds: instanceIds, - }) - if err != nil { - return resources, err - } + for { + ssmOutput, err := ssmClient.DescribeInstanceInformation(ctx, &ssm.DescribeInstanceInformationInput{ + NextToken: nexttoken, + }) + if err != nil { + return resources, err + } - account, accountID, err := fetchID(ctx, client) - if err != nil { - return resources, err - } + instanceIds := make([]string, 0, len(ssmOutput.InstanceInformationList)) + for _, ec2instance := range ssmOutput.InstanceInformationList { + instanceIds = append(instanceIds, *ec2instance.InstanceId) + } + ec2Output, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: instanceIds, + }) + if err != nil { + return resources, err + } + + account, accountID, err := fetchID(ctx, client) + if err != nil { + return resources, err + } - for _, ec2instance := range ec2Output.Reservations { - for _, instance := range ec2instance.Instances { + for _, ec2instance := range ec2Output.Reservations { + for _, instance := range ec2instance.Instances { - if instance.State.Name == types.InstanceStateNameRunning { - tags := make([]models.Tag, 0) - for _, tag := range instance.Tags { - tags = append(tags, models.Tag{ - Key: *tag.Key, - Value: *tag.Value, + if instance.State.Name == types.InstanceStateNameRunning { + tags := make([]models.Tag, 0) + for _, tag := range instance.Tags { + tags = append(tags, models.Tag{ + Key: *tag.Key, + Value: *tag.Value, + }) + } + + resources = append(resources, models.Resource{ + Provider: "AWS", + Account: account, + AccountId: accountID, + Service: "SSM Instance", + Region: client.AWSClient.Region, + ResourceId: *instance.InstanceId, + Name: string(instance.InstanceType), + CreatedAt: *instance.LaunchTime, + FetchedAt: time.Now(), + Tags: tags, + Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", + client.AWSClient.Region, client.AWSClient.Region, *instance.InstanceId), }) } - resources = append(resources, models.Resource{ - Provider: "AWS", - Account: account, - AccountId: accountID, - Service: "SSM-Instance", - Region: client.AWSClient.Region, - ResourceId: *instance.InstanceId, - Name: string(instance.InstanceType), - CreatedAt: *instance.LaunchTime, - FetchedAt: time.Now(), - Tags: tags, - Link: fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/home?region=%s#InstanceDetails:instanceId=%s", - client.AWSClient.Region, client.AWSClient.Region, *instance.InstanceId), - }) } - } + if ssmOutput.NextToken == nil { + break + } + nexttoken = ssmOutput.NextToken } return resources, nil From 852c0d6aeb4ae01b2fb9c5fea626ee698e044694 Mon Sep 17 00:00:00 2001 From: horiodino Date: Tue, 23 Jan 2024 23:08:47 +0530 Subject: [PATCH 09/10] fixed the failing CI --- providers/aws/aws.go | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/aws/aws.go b/providers/aws/aws.go index a5d740ad2..d78ba6660 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -100,6 +100,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.KeyPairs, ec2.PlacementGroups, systemsmanager.MaintenanceWindows, + systemsmanager.GetManagedEc2, ec2.VpcEndpoints, ec2.VpcPeeringConnections, kinesis.Streams, From 3c1b7b7d78718fa609f9a996f8c1f454de9aaeed Mon Sep 17 00:00:00 2001 From: Praful Khanduri <99384392+Horiodino@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:13:02 +0530 Subject: [PATCH 10/10] Update providers/aws/systemsmanager/managedinstances.go Co-authored-by: Azanul Haque <42029519+Azanul@users.noreply.github.com> --- providers/aws/systemsmanager/managedinstances.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/aws/systemsmanager/managedinstances.go b/providers/aws/systemsmanager/managedinstances.go index 41e96e84f..4fc665a0c 100644 --- a/providers/aws/systemsmanager/managedinstances.go +++ b/providers/aws/systemsmanager/managedinstances.go @@ -13,7 +13,7 @@ import ( "github.com/tailwarden/komiser/providers" ) -func getManagedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { +func GetManagedEc2(ctx context.Context, client providers.ProviderClient) ([]models.Resource, error) { resources := make([]models.Resource, 0) ssmClient := ssm.NewFromConfig(*client.AWSClient)