diff --git a/.gitignore b/.gitignore index 5a3f3c4..34bf507 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Temporary files *~ .*~ +\#*\# # Binaries for programs and plugins *.exe diff --git a/overlord.go b/overlord.go index 0a86d8e..a32d87f 100644 --- a/overlord.go +++ b/overlord.go @@ -13,12 +13,15 @@ import ( "time" "errors" "context" - + "github.com/AirVantage/overlord/pkg/lookable" + "github.com/AirVantage/overlord/pkg/resource" + "github.com/AirVantage/overlord/pkg/changes" + "github.com/AirVantage/overlord/pkg/state" "github.com/AirVantage/overlord/pkg/set" "github.com/BurntSushi/toml" - + "github.com/aws/smithy-go" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -32,63 +35,33 @@ var ( ipv6 = flag.Bool("ipv6", false, "Look for IPv6 addresses instead of IPv4") ) -type ResourceConfig struct { - Resource Resource `toml:"template"` -} - -type Resource struct { - Src string - Dest string - Groups []lookable.AutoScalingGroup - Tags []lookable.Tag - Subnets []lookable.Subnet - ReloadCmd string `toml:"reload_cmd"` -} - -type State map[string]set.Strings - -// Changes keeps track of added/removed IPs for a Resource. -// We store IPs as strings to support both IPv4 and IPv6. -type Changes struct { - addedIPs set.Strings - removedIPs set.Strings -} - -// NewChanges return a pointer to an initialized Changes struct. -func NewChanges() *Changes { - return &Changes{ - addedIPs: set.NewStringSet(), - removedIPs: set.NewStringSet(), - } -} - -func iterate(ctx context.Context, cfg aws.Config, state State) State { +func iterate(ctx context.Context, cfg aws.Config, prevState *state.State) *state.State { var ( - resources map[lookable.Lookable][]*Resource = make(map[lookable.Lookable][]*Resource) - resourcesToUpdate map[*Resource]*Changes = make(map[*Resource]*Changes) - newState State = make(State) + resources map[lookable.Lookable][]*resource.Resource = make(map[lookable.Lookable][]*resource.Resource) + resourcesToUpdate map[*resource.Resource]*changes.Changes[string] = make(map[*resource.Resource]*changes.Changes[string]) + newState *state.State = state.New() ) // log.Println("Start iteration") - + //load resources definition files resourcesDir, err := os.Open(filepath.Join(*configRoot, resourcesDirName)) defer func() { resourcesDir.Close() }() if err != nil { log.Fatal(err) } - + resourcesFiles, err := resourcesDir.Readdir(0) if err != nil { log.Fatal(err) } - + for _, resourceFile := range resourcesFiles { if filepath.Ext(resourceFile.Name()) != ".toml" || resourceFile.IsDir() { continue } - var rc *ResourceConfig + var rc *resource.ResourceConfig _, err := toml.DecodeFile(filepath.Join(*configRoot, resourcesDirName, resourceFile.Name()), &rc) if err != nil { log.Fatal(err) @@ -96,6 +69,12 @@ func iterate(ctx context.Context, cfg aws.Config, state State) State { log.Println("Read File", resourceFile.Name(), ":", rc) + rc.Resource.SrcFSInfo, err = os.Stat( filepath.Join(*configRoot, templatesDirName, rc.Resource.Src) ) + if err != nil { + log.Fatal(err) + } + newState.Templates[rc.Resource.Src] = &rc.Resource + // Store each resource in a reverse map, listing resource linked to each lookable to easily match updates need per lookable changes for _, group := range rc.Resource.Groups { resources[group] = append( resources[group], &rc.Resource) @@ -124,51 +103,66 @@ func iterate(ctx context.Context, cfg aws.Config, state State) State { var oe *smithy.OperationError if errors.As(err, &oe) { log.Fatal("Failed service call processing ..: service ", oe.Service(), ", operation: ", oe.Operation(), ", error: ", oe.Unwrap()) - + } else { log.Fatal("Error processing ..:", err.Error()) } } - newState[group] = set.NewStringSet() - changes := NewChanges() + newState.Ipsets[group] = set.New[string]() + changes := changes.New[string]() changed := false - if _, exists := state[group]; !exists { - state[group] = set.NewStringSet() + if _, exists := prevState.Ipsets[group]; !exists { + prevState.Ipsets[group] = set.New[string]() } + for _, ip := range ips { - newState[group].Add(ip) - if !state[group].Has(ip) { + newState.Ipsets[group].Add(ip) + if !prevState.Ipsets[group].Has(ip) { changed = true - changes.addedIPs.Add(ip) + changes.Add(ip) log.Println("For group", group, "new IP:", ip) } } - for _, oldIP := range state[group].ToSlice() { - if !newState[group].Has(oldIP) { + for _, oldIP := range prevState.Ipsets[group].ToSlice() { + if !newState.Ipsets[group].Has(oldIP) { changed = true - changes.removedIPs.Add(oldIP) + changes.Remove(oldIP) log.Println("For group", group, "deprecated IP:", oldIP) } } - // handle template file change ? - // handle resource added with existing lookable ? if changed { for _, resource := range resourcesset { log.Println("For group", group, "update ressource:", resource) - resourcesToUpdate[resource] = changes + + // Merge Changes to store IP changes across differents aws resources: + if prevChanges, exists := resourcesToUpdate[resource]; exists { + resourcesToUpdate[resource] = prevChanges.Merge(changes) + } else { + resourcesToUpdate[resource] = changes + } + } + } + } + + // If new resource or template file changed since last run: + for file, rc := range newState.Templates { + if prevrc, exists := prevState.Templates[file]; !exists || rc.SrcFSInfo.ModTime().Sub(prevrc.SrcFSInfo.ModTime()) > 0 { + log.Println("Template", file, "changed:", rc.SrcFSInfo.ModTime() ) + if _, exists := resourcesToUpdate[rc]; !exists { + resourcesToUpdate[rc] = changes.New[string]() } } } // Convert set to sorted array for use with text/template ips := make(map[string][]string) - for group, ipsSet := range newState { - ipsList := make([]string, 0, len(ipsSet)) - for ip := range ipsSet { + for group, ipsSet := range newState.Ipsets { + ipsList := make([]string, 0, len(*ipsSet)) + for ip := range *ipsSet { ipsList = append(ipsList, ip) } sort.Strings(ipsList) @@ -206,7 +200,7 @@ func iterate(ctx context.Context, cfg aws.Config, state State) State { //cmd := exec.Command(cmdSplit[0], cmdSplit[1:]...) cmd := exec.Command("bash", "-c", resource.ReloadCmd) if changes != nil { - cmd.Env = append(os.Environ(), mkEnvVar("IP_ADDED", changes.addedIPs.ToSlice()), mkEnvVar("IP_REMOVED", changes.removedIPs.ToSlice())) + cmd.Env = append(os.Environ(), mkEnvVar("IP_ADDED", changes.Added()), mkEnvVar("IP_REMOVED", changes.Removed())) } log.Println(cmd) err = cmd.Start() @@ -229,10 +223,10 @@ func iterate(ctx context.Context, cfg aws.Config, state State) State { func main() { var ( - syslogCfg string - cfg aws.Config - ctx context.Context = context.TODO() - runningState State = make(State) + syslogCfg string + cfg aws.Config + ctx context.Context = context.TODO() + runningState *state.State = state.New() ) log.SetFlags(0) @@ -252,7 +246,7 @@ func main() { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { log.Fatal(err) - } + } flag.Parse() diff --git a/pkg/changes/changes.go b/pkg/changes/changes.go new file mode 100644 index 0000000..6796ffb --- /dev/null +++ b/pkg/changes/changes.go @@ -0,0 +1,63 @@ +package changes + +import ( + "github.com/AirVantage/overlord/pkg/set" +) + +// Changes keeps track of added/removed IPs for a Resource. +// We store IPs as strings to support both IPv4 and IPv6. +type Changes[T comparable] struct { + addedIPs *set.Set[T] + removedIPs *set.Set[T] +} + +// NewChanges return a pointer to an initialized Changes struct. +func New[T comparable]() *Changes[T] { + return &Changes[T]{ + addedIPs: set.New[T](), + removedIPs: set.New[T](), + } +} + + +// Log changes +func (c *Changes[T])Add(add T) { + c.addedIPs.Add(add) +} +func (c *Changes[T])Remove(rem T) { + c.removedIPs.Add(rem) +} + +// Return changes as slice +func (c *Changes[T])Added() []T { + return c.addedIPs.ToSlice() +} +func (c *Changes[T])Removed() []T { + return c.removedIPs.ToSlice() +} + +// Return a deep copy of current object +func (c *Changes[T])Copy() *Changes[T] { + var copy *Changes[T] = New[T]() + + for _, added := range c.addedIPs.ToSlice() { + copy.addedIPs.Add(added) + } + for _, removed := range c.removedIPs.ToSlice() { + copy.removedIPs.Add(removed) + } + return copy +} + +// NewChanges return a pointer to an initialized Changes struct. +func (c *Changes[T])Merge(m *Changes[T]) *Changes[T] { + var merged *Changes[T] = c.Copy() + + for _, added := range m.addedIPs.ToSlice() { + merged.addedIPs.Add(added) + } + for _, removed := range m.removedIPs.ToSlice() { + merged.removedIPs.Add(removed) + } + return merged +} diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go new file mode 100644 index 0000000..563403d --- /dev/null +++ b/pkg/resource/resource.go @@ -0,0 +1,23 @@ +package resource +// Configuration file structure + +import ( + "os" + + "github.com/AirVantage/overlord/pkg/lookable" +) + + +type ResourceConfig struct { + Resource Resource `toml:"template"` +} + +type Resource struct { + Src string + Dest string + Groups []lookable.AutoScalingGroup + Tags []lookable.Tag + Subnets []lookable.Subnet + ReloadCmd string `toml:"reload_cmd"` + SrcFSInfo os.FileInfo +} diff --git a/pkg/set/strings.go b/pkg/set/set.go similarity index 51% rename from pkg/set/strings.go rename to pkg/set/set.go index a38db84..1890ea9 100644 --- a/pkg/set/strings.go +++ b/pkg/set/set.go @@ -1,29 +1,32 @@ package set +// Generic set data structure // Strings is a set of unique strings. -type Strings map[string]struct{} +type Set[T comparable] map[T]struct{} -// NewStringSet instantiates a new set of strings. -func NewStringSet() Strings { - return make(Strings) +// NewStringSet instantiates a new generic Set . +func New[T comparable]() *Set[T] { + s := make(Set[T]) + return &s } // Add a string to the set. -func (ss Strings) Add(s string) { +func (ss Set[T]) Add(s T) { if _, exists := ss[s]; !exists { ss[s] = struct{}{} } } // Has returns true if a strings is part of the set. -func (ss Strings) Has(s string) bool { +func (ss Set[T]) Has(s T) bool { _, exists := ss[s] return exists } // ToSlice returns a copy the set as a slice of strings. -func (ss Strings) ToSlice() []string { - slice := make([]string, 0, len(ss)) +func (ss Set[T]) ToSlice() []T { + // Allocate a large enough slice + slice := make([]T, 0, len(ss)) for key, _ := range ss { slice = append(slice, key) diff --git a/pkg/set/set_test.go b/pkg/set/set_test.go new file mode 100644 index 0000000..ea25a1b --- /dev/null +++ b/pkg/set/set_test.go @@ -0,0 +1,130 @@ +package set + +import ( + "testing" + "strconv" +) + +func TestGenSetString(t *testing.T) { + + cases := []struct { + init func(t *testing.T) *Set[string] + has string + expect bool + len int + }{ + /* Empty set */ + { + init: func(t *testing.T) *Set[string] { + return New[string]() + }, + has: "12", + expect: false, + len: 0, + + }, + /* One element */ + { + init: func(t *testing.T) *Set[string] { + ss := New[string]() + ss.Add("12") + return ss + }, + has: "12", + expect: true, + len: 1, + }, + /* Two elements */ + { + init: func(t *testing.T) *Set[string] { + ss := New[string]() + ss.Add("12") + ss.Add("13") + return ss + }, + has: "12", + expect: true, + len: 2, + }, + /* Two element, three adds */ + { + init: func(t *testing.T) *Set[string] { + ss := New[string]() + ss.Add("12") + ss.Add("12") + ss.Add("13") + return ss + }, + has: "12", + expect: true, + len: 2, + }, + } + + for i, tt := range cases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + + data := tt.init(t) + + if output := data.Has(tt.has); output != tt.expect { + t.Errorf("expect %v, got %v", tt.expect, output) + } + + if output := data.ToSlice(); len(output) != tt.len { + t.Errorf("expect %v, got %v", tt.len, output) + } + }) + } +} + +func TestGenSetInt(t *testing.T) { + + cases := []struct { + init func(t *testing.T) *Set[int] + has int + expect bool + len int + }{ + /* Single instance result */ + { + init: func(t *testing.T) *Set[int] { + return New[int]() + }, + has: 12, + expect: false, + len: 0, + + }, + /* Single instance result */ + { + init: func(t *testing.T) *Set[int] { + ss := New[int]() + ss.Add(12) + return ss + }, + has: 12, + expect: true, + len: 1, + + }, + } + + for i, tt := range cases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + + data := tt.init(t) + + if output := data.Has(tt.has); output != tt.expect { + t.Errorf("expect %v, got %v", tt.expect, output) + } + + if output := data.ToSlice(); len(output) != tt.len { + t.Errorf("expect %v, got %v", tt.len, output) + } + }) + } + +} +/* + +*/ diff --git a/pkg/set/strings_test.go b/pkg/set/strings_test.go deleted file mode 100644 index 2d0eed3..0000000 --- a/pkg/set/strings_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package set - -import ( - "testing" - "strconv" -) - -func TestStringSet(t *testing.T) { - - cases := []struct { - init func(t *testing.T) *Strings - has string - expect bool - len int - }{ - /* Single instance result */ - { - init: func(t *testing.T) *Strings { - ss := NewStringSet() - return &ss - }, - has: "12", - expect: false, - len: 0, - - }, - /* Single instance result */ - { - init: func(t *testing.T) *Strings { - ss := NewStringSet() - ss.Add("12") - return &ss - }, - has: "12", - expect: true, - len: 1, - }, - } - - for i, tt := range cases { - t.Run(strconv.Itoa(i), func(t *testing.T) { - - data := tt.init(t) - - if output := data.Has(tt.has); output != tt.expect { - t.Errorf("expect %v, got %v", tt.expect, output) - } - - if output := data.ToSlice(); len(output) != tt.len { - t.Errorf("expect %v, got %v", tt.len, output) - } - }) - } - -} -/* - -*/ diff --git a/pkg/state/state.go b/pkg/state/state.go new file mode 100644 index 0000000..a2590c7 --- /dev/null +++ b/pkg/state/state.go @@ -0,0 +1,21 @@ +package state + +import ( + "github.com/AirVantage/overlord/pkg/resource" + "github.com/AirVantage/overlord/pkg/set" +) + + +type State struct { + Ipsets map[string]*set.Set[string] + Templates map[string]*resource.Resource +} + +// NewChanges return a pointer to an initialized Changes struct. +func New() *State { + return &State{ + Ipsets: make(map[string]*set.Set[string]), + Templates: make(map[string]*resource.Resource), + } +} + diff --git a/tests/config1/resources/test2.toml b/tests/config1/resources/test2.toml index a5b2da9..5916802 100644 --- a/tests/config1/resources/test2.toml +++ b/tests/config1/resources/test2.toml @@ -2,5 +2,5 @@ src = "test2.conf.tmpl" dest = "tests/config1/output/test2.conf" tags = [ "qa-nat-1", "qa-nat-2" ] -groups = [ "qa-av-routing-asg", "qa-av-device-lw-asg", "qa-lwm2mbs-asg", "qa-mqttfe-dmz-asg", "dev1-av-routing-asg" ] +groups = [ "qa-av-routing-asg", "qa-av-device-lw-asg", "qa-lwm2mbs-asg", "qa-mqttfe-dmz-asg" ] reload_cmd = "/bin/true"