diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e53ed16..15782e8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,7 +94,7 @@ jobs: command: | terraform init - terraform apply -var tags="{\"build_url\": \"$CIRCLE_BUILD_URL\"}" -auto-approve -var launch_type=<> + terraform apply -var tags="{\"build_url\": \"$CIRCLE_BUILD_URL\", \"build_time\": \"$(date +%s)\"}" -auto-approve -var launch_type=<> # Restore go module cache if there is one - restore_cache: diff --git a/cleanup/ec2.go b/cleanup/ec2.go new file mode 100644 index 00000000..7ae93d1d --- /dev/null +++ b/cleanup/ec2.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +// EC2Instances implements the Resource interface +type EC2Instances struct { + ids []string + cfg aws.Config +} + +func (e EC2Instances) String() string { + return fmt.Sprint(e.ids) +} + +func (e EC2Instances) Delete() error { + log.Printf("Delete EC2 Instances: %v", e.ids) + return nil +} + +func (e EC2Instances) Wait() error { + log.Printf("Wait for EC2 Instances: %v", e.ids) + return nil +} + +func ListEC2Instances(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Println("Listing EC2 instances") + ec2Client := ec2.NewFromConfig(cfg) + instances, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", buildUrlTagName)), + Values: []string{buildUrlPrefixPlusStar}, + }, + { + Name: aws.String("tag:Name"), + Values: []string{"consul-ecs-*"}, + }, + { + Name: aws.String("instance-state-name"), + Values: []string{"running"}, + }, + }, + }) + if err != nil { + return err + } + + var ids []string + for _, rsv := range instances.Reservations { + for _, inst := range rsv.Instances { + if isOldBuildTime(getTag(buildTimeTagName, inst.Tags)) { + ids = append(ids, *inst.InstanceId) + } + } + } + if len(ids) > 0 { + resourceChan <- EC2Instances{ + ids: ids, + cfg: cfg, + } + } + return nil +} diff --git a/cleanup/ecs.go b/cleanup/ecs.go new file mode 100644 index 00000000..9a624908 --- /dev/null +++ b/cleanup/ecs.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ecs" + ecsTypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" + "github.com/hashicorp/go-multierror" +) + +type ECSCluster struct { + arn string + services []string + cfg aws.Config +} + +func (e ECSCluster) String() string { + return fmt.Sprintf("arn=%s services=%v", e.arn, e.services) +} + +func (e ECSCluster) Delete() error { + panic("implement me") +} + +func (e ECSCluster) Wait() error { + panic("implement me") +} + +func ListECSClusters(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Println("Listing ECS clusters") + ecsClient := ecs.NewFromConfig(cfg) + list, err := ecsClient.ListClusters(ctx, nil) + if err != nil { + return err + } + + var allConsulEcsArns []string + for _, arn := range list.ClusterArns { + if strings.Contains(arn, "consul-ecs") { + allConsulEcsArns = append(allConsulEcsArns, arn) + } + } + + describe, err := ecsClient.DescribeClusters(ctx, &ecs.DescribeClustersInput{ + Clusters: allConsulEcsArns, + Include: []ecsTypes.ClusterField{ecsTypes.ClusterFieldTags}, + }) + if err != nil { + return err + } + + var clustersToCleanup []*ECSCluster + for _, cluster := range describe.Clusters { + if *cluster.Status == "INACTIVE" { + continue + } + buildUrl := getTag(buildUrlTagName, cluster.Tags) + buildTimestamp := getTag(buildTimeTagName, cluster.Tags) + if strings.Contains(buildUrl, buildUrlPrefix) && isOldBuildTime(buildTimestamp) { + clustersToCleanup = append(clustersToCleanup, &ECSCluster{arn: *cluster.ClusterArn}) + } + } + + // Services in the cluster must be deleted before the cluster can be deleted. + var errors error + for _, c := range clustersToCleanup { + log.Printf("Listing ECS services for cluster=%s", c.arn) + services, err := ecsClient.ListServices(ctx, &ecs.ListServicesInput{ + Cluster: aws.String(c.arn), + }) + if err != nil { + errors = multierror.Append(errors, err) + } else { + c.services = services.ServiceArns + } + } + + for _, c := range clustersToCleanup { + resourceChan <- *c + } + return errors +} diff --git a/cleanup/elastic-ip.go b/cleanup/elastic-ip.go new file mode 100644 index 00000000..774fb92c --- /dev/null +++ b/cleanup/elastic-ip.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +type ElasticIP struct { + allocationId string +} + +func (e ElasticIP) String() string { + return e.allocationId +} + +func (e ElasticIP) Delete() error { + panic("implement me") +} + +func (e ElasticIP) Wait() error { + panic("implement me") +} + +func ListElasticIPs(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Println("Listing elastic ips") + ec2Client := ec2.NewFromConfig(cfg) + addresses, err := ec2Client.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", buildUrlTagName)), + Values: []string{buildUrlPrefixPlusStar}, + }, + { + Name: aws.String("tag:Name"), + Values: []string{"consul-ecs-*"}, + }, + }, + }) + if err != nil { + return err + } + + // NOTE: May be associated (to the NAT gateway). If the NAT GW is deleted first, + // it's okay, and we ensure that ordering elsewhere. + for _, addr := range addresses.Addresses { + if isOldBuildTime(getTag(buildTimeTagName, addr.Tags)) { + resourceChan <- ElasticIP{allocationId: *addr.AllocationId} + } + } + return nil +} diff --git a/cleanup/go.mod b/cleanup/go.mod new file mode 100644 index 00000000..a647c0e1 --- /dev/null +++ b/cleanup/go.mod @@ -0,0 +1,25 @@ +module cleanup + +go 1.17 + +require ( + github.com/aws/aws-sdk-go-v2 v1.10.0 + github.com/aws/aws-sdk-go-v2/config v1.9.0 + github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.8.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.20.0 + github.com/aws/aws-sdk-go-v2/service/ecs v1.10.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.11.0 + github.com/hashicorp/go-multierror v1.1.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.8.0 // indirect + github.com/aws/smithy-go v1.8.1 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/cleanup/go.sum b/cleanup/go.sum new file mode 100644 index 00000000..ce64f606 --- /dev/null +++ b/cleanup/go.sum @@ -0,0 +1,47 @@ +github.com/aws/aws-sdk-go-v2 v1.10.0 h1:+dCJ5W2HiZNa4UtaIc5ljKNulm0dK0vS5dxb5LdDOAA= +github.com/aws/aws-sdk-go-v2 v1.10.0/go.mod h1:U/EyyVvKtzmFeQQcca7eBotKdlpcP2zzU6bXBYcf7CE= +github.com/aws/aws-sdk-go-v2/config v1.9.0 h1:SkREVSwi+J8MSdjhJ96jijZm5ZDNleI0E4hHCNivh7s= +github.com/aws/aws-sdk-go-v2/config v1.9.0/go.mod h1:qhK5NNSgo9/nOSMu3HyE60WHXZTWTHTgd5qtIF44vOQ= +github.com/aws/aws-sdk-go-v2/credentials v1.5.0 h1:r6470olsn2qyOe2aLzK6q+wfO3dzNcMujRT3gqBgBB8= +github.com/aws/aws-sdk-go-v2/credentials v1.5.0/go.mod h1:kvqTkpzQmzri9PbsiTY+LvwFzM0gY19emlAWwBOJMb0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.7.0 h1:FKaqk7geL3oIqSwGJt5SWUKj8uJ+qLZNqlBuqq6sFyA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.7.0/go.mod h1:KqEkRkxm/+1Pd/rENRNbQpfblDBYeg5HDSqjB6ks8hA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5 h1:zPxLGWALExNepElO0gYgoqsbqTlt4ZCrhZ7XlfJ+Qlw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.5/go.mod h1:6ZBTuDmvpCOD4Sf1i2/I3PgftlEcDGgvi8ocq64oQEg= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.8.0 h1:WQY4Qit9/XiIAH4mhbT8qaMFIWl2OExcXrFecz2eGXQ= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.8.0/go.mod h1:clJaIaj1E6A3vSq1Qd++dfcG3dbF8gUfTEUiaTZxpVY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.20.0 h1:qvcoul6cfXEjiQMY1N43zaDui3FWsEpXLVxHlmWc3pk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.20.0/go.mod h1:P+gshV4VLT7jUbWALAhV9lXDyZ40R7E/Rvr2ryBqn2s= +github.com/aws/aws-sdk-go-v2/service/ecs v1.10.0 h1:C1RlobZmzZ0R3N+csY7eTNEGFjjTp3fKWTiyu6XjhKc= +github.com/aws/aws-sdk-go-v2/service/ecs v1.10.0/go.mod h1:WhlAAJ0XsFjT4Xr2CYaSfOkxWV6Zv9nptqw/cxDxxC4= +github.com/aws/aws-sdk-go-v2/service/iam v1.11.0 h1:RLDJKse1N4HkYQ+PLse7UzAHC7AnTEkG/hXEBE5Arm8= +github.com/aws/aws-sdk-go-v2/service/iam v1.11.0/go.mod h1:HILqe6vfjMKnuUO64jXXFAcLBQ5sT2P7xNQiXy6q7BM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.4.0 h1:/T5wKsw/po118HEDvnSE8YU7TESxvZbYM2rnn+Oi7Kk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.4.0/go.mod h1:X5/JuOxPLU/ogICgDTtnpfaQzdQJO0yKDcpoxWLLJ8Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.5.0 h1:VnrCAJTp1bDxU79UuW/D4z7bwZ7xOc7JjDKpqXL/m04= +github.com/aws/aws-sdk-go-v2/service/sso v1.5.0/go.mod h1:GsqaJOJeOfeYD88/2vHWKXegvDRofDqWwC5i48A2kgs= +github.com/aws/aws-sdk-go-v2/service/sts v1.8.0 h1:7N7RsEVvUcvEg7jrWKU5AnSi4/6b6eY9+wG1g6W4ExE= +github.com/aws/aws-sdk-go-v2/service/sts v1.8.0/go.mod h1:dOlm91B439le5y1vtPCk5yJtbx3RdT3hRGYRY8TYKvQ= +github.com/aws/smithy-go v1.8.1 h1:9Y6qxtzgEODaLNGN+oN2QvcHvKUe4jsH8w4M+8LXzGk= +github.com/aws/smithy-go v1.8.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/cleanup/iam.go b/cleanup/iam.go new file mode 100644 index 00000000..8262bfe6 --- /dev/null +++ b/cleanup/iam.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/hashicorp/go-multierror" +) + +type IamRole struct { + id string + name string + instanceProfileNames []string + managedPoliciesAtttached string +} + +func (i IamRole) String() string { + return fmt.Sprintf("id=%s name=%s instanceProfiles=%s", + i.id, i.name, i.instanceProfileNames) +} + +func (i IamRole) Delete() error { + panic("implement me") +} + +func (i IamRole) Wait() error { + panic("implement me") +} + +func ListIamRoles(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Printf("Listing IAM roles") + + iamClient := iam.NewFromConfig(cfg) + pager := iam.NewListRolesPaginator(iamClient, nil) + + var errors error + + // This is horrible. There's no way to filter by name. And we seem to hit rate limits + // all the time in our account. + for pager.HasMorePages() { + page, err := pager.NextPage(ctx) + if err != nil { + return err + } + + for _, role := range page.Roles { + if strings.Index(*role.RoleName, "consul-ecs") != 0 { + continue + } + + tags, err := iamClient.ListRoleTags(ctx, &iam.ListRoleTagsInput{ + RoleName: role.RoleName, + }) + if err != nil { + errors = multierror.Append(errors, err) + continue + } + + buildUrl := getTag(buildUrlTagName, tags.Tags) + buildTime := getTag(buildTimeTagName, tags.Tags) + if strings.Contains(buildUrl, buildUrlPrefix) && isOldBuildTime(buildTime) { + resourceChan <- IamRole{ + id: *role.RoleId, + name: *role.RoleName, + instanceProfileNames: listInstanceProfiles(iamClient, ctx, *role.RoleName), + // managedPoliciesAtttached: "", + } + } + } + + } + return errors +} + +func listInstanceProfiles(iamClient *iam.Client, ctx context.Context, roleName string) []string { + log.Printf("Listing instance profiles for role %s", roleName) + profiles, err := iamClient.ListInstanceProfilesForRole(ctx, &iam.ListInstanceProfilesForRoleInput{ + RoleName: aws.String(roleName), + }) + if err != nil { + log.Printf("warning: error listing instance profiles for role: %s", err) + return nil + } + + var profileNames []string + for _, profile := range profiles.InstanceProfiles { + profileNames = append(profileNames, *profile.InstanceProfileName) + } + return profileNames +} diff --git a/cleanup/log-group.go b/cleanup/log-group.go new file mode 100644 index 00000000..16d41720 --- /dev/null +++ b/cleanup/log-group.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/hashicorp/go-multierror" +) + +type LogGroup struct { + name string +} + +func (l LogGroup) String() string { + return l.name +} + +func (l LogGroup) Delete() error { + panic("implement me") +} + +func (l LogGroup) Wait() error { + panic("implement me") +} + +func ListLogGroups(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + cwlClient := cloudwatchlogs.NewFromConfig(cfg) + groups, err := cwlClient.DescribeLogGroups(ctx, &cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: aws.String("consul-ecs"), + }) + if err != nil { + return err + } + + var errors error + for _, group := range groups.LogGroups { + groupTags, err := cwlClient.ListTagsLogGroup(ctx, &cloudwatchlogs.ListTagsLogGroupInput{ + LogGroupName: group.LogGroupName, + }) + if err != nil { + errors = multierror.Append(errors, err) + continue + } + + buildUrl := groupTags.Tags[buildUrlTagName] + buildTime := groupTags.Tags[buildTimeTagName] + if strings.Contains(buildUrl, buildUrlPrefix) && isOldBuildTime(buildTime) { + resourceChan <- LogGroup{name: *group.LogGroupName} + } + } + return errors +} diff --git a/cleanup/main.go b/cleanup/main.go new file mode 100644 index 00000000..07ab24eb --- /dev/null +++ b/cleanup/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "log" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" +) + +const ( + // AWS supports patterns for filtering on tags, but not on all APIs. + buildUrlTagName = "build_url" + // buildUrlPrefix = "https://circleci.com/gh/hashicorp/terraform-aws-consul-ecs/" + buildUrlPrefix = "http://test.example" + buildUrlPrefixPlusStar = buildUrlPrefix + "*" + + // Only cleanup resources at least this old. + // We rely on a tag for this since not all resources have a creation time. + buildTimeTagName = "build_time" + // resourceAge = 4 * 24 * time.Hour + resourceAge = 30 * time.Second +) + +type Resource interface { + String() string + Delete() error + Wait() error +} + +type ResourceListFn func(aws.Config, context.Context, chan Resource) error + +func main() { + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + if err != nil { + log.Fatal(err) + } + + resources := ListResources(cfg) + for _, r := range resources { + log.Printf("%T %v", r, r) + } +} + +func ListResources(cfg aws.Config) []Resource { + ctx := context.Background() + resourceChan := make(chan Resource, 10000) + var wg sync.WaitGroup + defer wg.Wait() + + listFns := []ResourceListFn{ + ListEC2Instances, + ListECSClusters, + ListNatGateways, + ListVPCs, + ListLogGroups, + ListElasticIPs, + // Expensive. No way to filter by name. And hits rate limits. + // ListIamRoles, + } + for _, fn := range listFns { + wg.Add(1) + go func(lister ResourceListFn) { + defer wg.Done() + if err := lister(cfg, ctx, resourceChan); err != nil { + log.Printf("error: %s", err) + } + }(fn) + } + + wg.Wait() + close(resourceChan) // so we can iterate over it + + var resources []Resource + for r := range resourceChan { + resources = append(resources, r) + } + return resources +} diff --git a/cleanup/nat-gateway.go b/cleanup/nat-gateway.go new file mode 100644 index 00000000..686fcb41 --- /dev/null +++ b/cleanup/nat-gateway.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +type NatGateway struct { + id string +} + +func (n NatGateway) String() string { + return n.id +} + +func (n NatGateway) Delete() error { + panic("implement me") +} + +func (n NatGateway) Wait() error { + panic("implement me") +} + +func ListNatGateways(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Println("Listing NAT gateways") + ec2Client := ec2.NewFromConfig(cfg) + gateways, err := ec2Client.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{ + Filter: []ec2Types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", buildUrlTagName)), + Values: []string{buildUrlPrefixPlusStar}, + }, + { + Name: aws.String("tag:Name"), + Values: []string{"consul-ecs-*"}, + }, + { + Name: aws.String("state"), + Values: []string{ + string(ec2Types.NatGatewayStateAvailable), + string(ec2Types.NatGatewayStateFailed), + string(ec2Types.NatGatewayStatePending), + }, + }, + }, + }) + if err != nil { + return err + } + + for _, gw := range gateways.NatGateways { + if isOldBuildTime(getTag(buildTimeTagName, gw.Tags)) { + resourceChan <- NatGateway{*gw.NatGatewayId} + } + } + return nil +} diff --git a/cleanup/tags.go b/cleanup/tags.go new file mode 100644 index 00000000..9f9e0fba --- /dev/null +++ b/cleanup/tags.go @@ -0,0 +1,55 @@ +package main + +import ( + "log" + "reflect" + "strconv" + "time" +) + +// Some reflection. The SDK has a separate but identical Tag struct for each service. +func getTag(lookupKey string, tags interface{}) string { + slice := reflect.ValueOf(tags) + if slice.Kind() != reflect.Slice { + log.Printf("getTag requires a slice (not %T)", tags) + return "" + } + + isPtrToString := func(v reflect.Value) bool { + return v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.String + } + + for i := 0; i < slice.Len(); i++ { + tag := slice.Index(i) + if tag.Kind() != reflect.Struct { + log.Printf("getTag requires a slice of structs (not []%T)", tag) + return "" + } + + key := tag.FieldByName("Key") + if !key.IsValid() || !isPtrToString(key) { + log.Printf("getTag requires a Tag struct with field `Key` of type *string (not %+v)", tag) + return "" + } + if lookupKey != key.Elem().String() { + continue + } + + val := tag.FieldByName("Value") + if !val.IsValid() || !isPtrToString(val) { + log.Printf("getTag requires a Tag struct with field `Value` of type *string (not %T)", key) + return "" + } + return val.Elem().String() + } + return "" +} + +func isOldBuildTime(timestamp string) bool { + if unixTime, err := strconv.Atoi(timestamp); err == nil { + buildTime := time.Unix(int64(unixTime), 0) + return buildTime.Before(time.Now().Add(-resourceAge)) + } + log.Printf("warning: unable to parse build time: %q", timestamp) + return false +} diff --git a/cleanup/vpc.go b/cleanup/vpc.go new file mode 100644 index 00000000..b0be4531 --- /dev/null +++ b/cleanup/vpc.go @@ -0,0 +1,161 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +type VPC struct { + id string + igwId string + subnetIds []string + securityGroupIds []string + routeTableIds []string +} + +func (v VPC) String() string { + return fmt.Sprintf("vpc=%s igw=%s subnets=%s secgroups=%s routetables=%s", + v.id, v.igwId, v.subnetIds, v.securityGroupIds, v.routeTableIds) +} + +func (v VPC) Delete() error { + panic("implement me") +} + +func (v VPC) Wait() error { + panic("implement me") +} + +func ListVPCs(cfg aws.Config, ctx context.Context, resourceChan chan Resource) error { + log.Println("Listing VPCs") + ec2Client := ec2.NewFromConfig(cfg) + describeVpcs, err := ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", buildUrlTagName)), + Values: []string{buildUrlPrefixPlusStar}, + }, + { + Name: aws.String("tag:Name"), + Values: []string{"consul-ecs-*"}, + }, + }, + }) + if err != nil { + return err + } + + for _, vpc := range describeVpcs.Vpcs { + if isOldBuildTime(getTag(buildTimeTagName, vpc.Tags)) { + resourceChan <- VPC{ + id: *vpc.VpcId, + igwId: getIgw(ec2Client, ctx, *vpc.VpcId), + subnetIds: listSubnets(ec2Client, ctx, *vpc.VpcId), + securityGroupIds: listSecurityGroups(ec2Client, ctx, *vpc.VpcId), + routeTableIds: listRouteTables(ec2Client, ctx, *vpc.VpcId), + } + } + } + return nil +} + +func getIgw(ec2Client *ec2.Client, ctx context.Context, vpcId string) string { + log.Printf("Listing internet gateways in VPC %s", vpcId) + gateways, err := ec2Client.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String("attachment.vpc-id"), + Values: []string{vpcId}, + }, + }, + }) + if err != nil { + log.Printf("warning: error listing internet gateways: %s", err) + return "" + } + + if len(gateways.InternetGateways) >= 1 { + return *gateways.InternetGateways[0].InternetGatewayId + } + return "" +} + +func listSubnets(ec2Client *ec2.Client, ctx context.Context, vpcId string) []string { + log.Printf("Listing subnets for VPC %s", vpcId) + subnets, err := ec2Client.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String("vpc-id"), + Values: []string{vpcId}, + }, + }, + }) + if err != nil { + log.Printf("warning: error listing subnets: %s", err) + return nil + } + + var subnetIds []string + for _, subnet := range subnets.Subnets { + subnetIds = append(subnetIds, *subnet.SubnetId) + } + return subnetIds +} + +func listSecurityGroups(ec2Client *ec2.Client, ctx context.Context, vpcId string) []string { + log.Printf("Listing security groups for VPC %s", vpcId) + groups, err := ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String("vpc-id"), + Values: []string{vpcId}, + }, + }, + }) + if err != nil { + log.Printf("warning: error listing security groups: %s", err) + return nil + } + + var groupIds []string + for _, group := range groups.SecurityGroups { + groupIds = append(groupIds, *group.GroupId) + } + return groupIds +} + +func listRouteTables(ec2Client *ec2.Client, ctx context.Context, vpcId string) []string { + log.Printf("Listing route tables for VPC %s", vpcId) + tables, err := ec2Client.DescribeRouteTables(ctx, &ec2.DescribeRouteTablesInput{ + Filters: []ec2Types.Filter{ + { + Name: aws.String("vpc-id"), + Values: []string{vpcId}, + }, + }, + }) + if err != nil { + log.Printf("warning: error listing security groups: %s", err) + return nil + } + + var routeTableIds []string + for _, table := range tables.RouteTables { + // Avoid deleting the main route table association. + isMain := false + for _, assoc := range table.Associations { + if assoc.Main != nil && *assoc.Main { + isMain = true + } + } + if !isMain { + routeTableIds = append(routeTableIds, *table.RouteTableId) + } + } + return routeTableIds +} diff --git a/test/acceptance/setup-terraform/ec2.tf b/test/acceptance/setup-terraform/ec2.tf index d6e32fea..a2df9268 100644 --- a/test/acceptance/setup-terraform/ec2.tf +++ b/test/acceptance/setup-terraform/ec2.tf @@ -28,6 +28,8 @@ resource "aws_iam_role" "instance_role" { managed_policy_arns = [ "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" ] + + tags = var.tags } resource "aws_iam_instance_profile" "instance_profile" {