Skip to content

Commit

Permalink
Merge pull request #114 from uselagoon/db-groups
Browse files Browse the repository at this point in the history 
Use Lagoon API DB to determine project group membership
  • Loading branch information
@smlx
smlx authored May 10, 2024
2 parents 9bce9ca + 30844f0 commit c52c193
Show file tree
Hide file tree
Showing 9 changed files with 239 additions and 147 deletions.
18 changes: 11 additions & 7 deletions cmd/lagoon-opensearch-sync/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,13 +86,17 @@ func (cmd *SyncCmd) Run(log *zap.Logger) error {
return nil
}
// continue running in a loop
tick := time.NewTicker(cmd.Period)
for range tick.C {
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects,
cmd.LegacyIndexPatternDelimiter)
if err != nil {
return err
ticker := time.NewTicker(cmd.Period)
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
err = sync.Sync(ctx, log, l, k, o, d, cmd.DryRun, cmd.Objects,
cmd.LegacyIndexPatternDelimiter)
if err != nil {
return err
}
}
}
return nil
}
36 changes: 34 additions & 2 deletions internal/lagoondb/client.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// Package lagoondb implements a client for the Lagoon API database.
package lagoondb

import (
Expand All @@ -20,6 +21,13 @@ type Project struct {
Name string `db:"name"`
}

// groupProjectMapping maps Lagoon group ID to project ID.
// This type is only used for database unmarshalling.
type groupProjectMapping struct {
GroupID string `db:"group_id"`
ProjectID int `db:"project_id"`
}

// ErrNoResult is returned by client methods if there is no result.
var ErrNoResult = errors.New("no rows in result set")

Expand All @@ -36,8 +44,7 @@ func NewClient(ctx context.Context, dsn string) (*Client, error) {
return &Client{db: db}, nil
}

// Projects returns the Environment associated with the given
// Namespace name (on Openshift this is the project name).
// Projects returns a slice of all Projects in the Lagoon API DB.
func (c *Client) Projects(ctx context.Context) ([]Project, error) {
// run query
var projects []Project
Expand All @@ -52,3 +59,28 @@ func (c *Client) Projects(ctx context.Context) ([]Project, error) {
}
return projects, nil
}

// GroupProjectsMap returns a map of Group (UU)IDs to Project IDs.
// This denotes Project Group membership in Lagoon.
func (c *Client) GroupProjectsMap(
ctx context.Context,
) (map[string][]int, error) {
var gpms []groupProjectMapping
err := c.db.SelectContext(ctx, &gpms, `
SELECT group_id, project_id
FROM kc_group_projects`)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNoResult
}
return nil, err
}
groupProjectsMap := map[string][]int{}
// no need to check for duplicates here since the table has:
// UNIQUE KEY `group_project` (`group_id`,`project_id`)
for _, gpm := range gpms {
groupProjectsMap[gpm.GroupID] =
append(groupProjectsMap[gpm.GroupID], gpm.ProjectID)
}
return groupProjectsMap, nil
}
54 changes: 40 additions & 14 deletions internal/sync/indexpatterns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,29 @@ func calculateIndexPatternDiff(log *zap.Logger,

// generateIndexPatternsForGroup returns a slice of index patterns for all the
// projects associated with the given group.
func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
projectNames map[int]string, legacyDelimiter bool) ([]string, error) {
pids, err := projectIDsForGroup(group)
if err != nil {
return nil, fmt.Errorf("couldn't get project IDs for group: %v", err)
func generateIndexPatternsForGroup(
log *zap.Logger,
group keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
legacyDelimiter bool,
) ([]string, error) {
pids, ok := groupProjectsMap[group.ID]
if !ok {
return nil, fmt.Errorf("missing project group ID %s in groupProjectsMap",
group.ID)
}
var indexPatterns []string
for _, pid := range pids {
name, ok := projectNames[pid]
if !ok {
log.Debug("invalid project ID in lagoon-projects group attribute",
// If you see this warning it means that a project ID appears in the
// kc_group_projects table that does not appear in the projects table in
// the Lagoon API DB.
// This is likely a bug in Lagoon causing loss of referential integrity,
// as there is no foreign key constraint to enforce valid project IDs in
// the group mapping.
log.Warn("invalid project ID when generating index patterns",
zap.Int("projectID", pid))
continue
}
Expand All @@ -156,17 +168,22 @@ func generateIndexPatternsForGroup(log *zap.Logger, group keycloak.Group,
//
// Only regular Lagoon groups are associated with a tenant (which is where
// index patterns are placed), so project groups are ignored.
func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,
projectNames map[int]string, legacyDelimiter bool) map[string]map[string]bool {
func generateIndexPatterns(
log *zap.Logger,
groups []keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
legacyDelimiter bool,
) map[string]map[string]bool {
indexPatterns := map[string]map[string]bool{}
var patterns []string
var err error
for _, group := range groups {
if !isLagoonGroup(group) || isProjectGroup(log, group) {
if !isLagoonGroup(group, groupProjectsMap) || isProjectGroup(log, group) {
continue
}
patterns, err = generateIndexPatternsForGroup(log, group, projectNames,
legacyDelimiter)
groupProjectsMap, legacyDelimiter)
if err != nil {
log.Warn("couldn't generate index patterns for group",
zap.String("group", group.Name), zap.Error(err))
Expand All @@ -191,17 +208,26 @@ func generateIndexPatterns(log *zap.Logger, groups []keycloak.Group,

// syncIndexPatterns reconciles Opensearch Dashboards index patterns with
// Lagoon logging requirements.
func syncIndexPatterns(ctx context.Context, log *zap.Logger,
groups []keycloak.Group, projectNames map[int]string, o OpensearchService,
d DashboardsService, dryRun, legacyDelimiter bool) {
func syncIndexPatterns(
ctx context.Context,
log *zap.Logger,
groups []keycloak.Group,
projectNames map[int]string,
groupProjectsMap map[string][]int,
o OpensearchService,
d DashboardsService,
dryRun,
legacyDelimiter bool,
) {
// get index patterns from Opensearch
existing, err := o.IndexPatterns(ctx)
if err != nil {
log.Error("couldn't get index patterns from Opensearch", zap.Error(err))
return
}
// generate the index patterns required by Lagoon
required := generateIndexPatterns(log, groups, projectNames, legacyDelimiter)
required := generateIndexPatterns(log, groups, projectNames,
groupProjectsMap, legacyDelimiter)
// calculate index templates to add/remove
toCreate, toDelete := calculateIndexPatternDiff(log, existing, required)
for tenant, patternIDMap := range toDelete {
Expand Down
54 changes: 25 additions & 29 deletions internal/sync/indexpatterns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ import (
)

type generateIndexPatternsForGroupInput struct {
group keycloak.Group
projectNames map[int]string
group keycloak.Group
projectNames map[int]string
groupProjectsMap map[string][]int
}

type generateIndexPatternsForGroupOutput struct {
Expand All @@ -31,10 +32,6 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
ID: "f6697da3-016a-43cd-ba9f-3f5b91b45302",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "drupal-example",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"drupal-example":[31,34,35]}`},
"lagoon-projects": {`31,34,35`},
},
},
},
projectNames: map[int]string{
Expand All @@ -43,6 +40,9 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
35: "drupal10-prerelease",
36: "delta-backend",
},
groupProjectsMap: map[string][]int{
"f6697da3-016a-43cd-ba9f-3f5b91b45302": {31, 34, 35},
},
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
Expand Down Expand Up @@ -71,10 +71,6 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "drupal-example2",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"drupal-example":[31,35,44]}`},
"lagoon-projects": {`31,35,44`},
},
},
},
projectNames: map[int]string{
Expand All @@ -83,6 +79,9 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
35: "drupal10-prerelease",
36: "delta-backend",
},
groupProjectsMap: map[string][]int{
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee": {31, 35, 44},
},
},
expect: generateIndexPatternsForGroupOutput{
indexPatterns: []string{
Expand All @@ -106,7 +105,7 @@ func TestGenerateIndexPatternsForGroup(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns, err := sync.GenerateIndexPatternsForGroup(log, tc.input.group,
tc.input.projectNames, false)
tc.input.projectNames, tc.input.groupProjectsMap, false)
if (err == nil && tc.expect.err != nil) ||
(err != nil && tc.expect.err == nil) {
tt.Fatalf("got err:\n%v\nexpected err:\n%v\n", err, tc.expect.err)
Expand Down Expand Up @@ -376,9 +375,10 @@ func TestCalculateIndexPatternDiff(t *testing.T) {

func TestGenerateIndexPatterns(t *testing.T) {
type generateIndexPatternsInput struct {
groups []keycloak.Group
projectNames map[int]string
legacyDelimiter bool
groups []keycloak.Group
projectNames map[int]string
groupProjectsMap map[string][]int
legacyDelimiter bool
}
var testCases = map[string]struct {
input generateIndexPatternsInput
Expand All @@ -391,27 +391,25 @@ func TestGenerateIndexPatterns(t *testing.T) {
ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "foocorp",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`},
"lagoon-projects": {`3133,34435`},
},
},
},
{
ID: "9f92af94-a7ee-4759-83bb-2b983bd30142",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "project-drupal12-base",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`},
"lagoon-projects": {`34435`},
"type": {`project-default-group`},
"type": {`project-default-group`},
},
},
},
},
projectNames: map[int]string{
34435: "drupal12-base",
},
groupProjectsMap: map[string][]int{
"08fef83d-cde7-43a5-8bd2-a18cf440214a": {3133, 34435},
"9f92af94-a7ee-4759-83bb-2b983bd30142": {34435},
},
legacyDelimiter: false,
},
expect: map[string]map[string]bool{
Expand Down Expand Up @@ -446,27 +444,25 @@ func TestGenerateIndexPatterns(t *testing.T) {
ID: "08fef83d-cde7-43a5-8bd2-a18cf440214a",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "foocorp",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"foocorp":[3133,34435]}`},
"lagoon-projects": {`3133,34435`},
},
},
},
{
ID: "9f92af94-a7ee-4759-83bb-2b983bd30142",
GroupUpdateRepresentation: keycloak.GroupUpdateRepresentation{
Name: "project-drupal12-base",
Attributes: map[string][]string{
"group-lagoon-project-ids": {`{"project-drupal12-base":[34435]}`},
"lagoon-projects": {`34435`},
"type": {`project-default-group`},
"type": {`project-default-group`},
},
},
},
},
projectNames: map[int]string{
34435: "drupal12-base",
},
groupProjectsMap: map[string][]int{
"08fef83d-cde7-43a5-8bd2-a18cf440214a": {3133, 34435},
"9f92af94-a7ee-4759-83bb-2b983bd30142": {34435},
},
legacyDelimiter: true,
},
expect: map[string]map[string]bool{
Expand Down Expand Up @@ -499,7 +495,7 @@ func TestGenerateIndexPatterns(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(tt *testing.T) {
indexPatterns := sync.GenerateIndexPatterns(
log, tc.input.groups, tc.input.projectNames,
log, tc.input.groups, tc.input.projectNames, tc.input.groupProjectsMap,
tc.input.legacyDelimiter)
if !reflect.DeepEqual(indexPatterns, tc.expect) {
tt.Fatalf("got:\n%v\nexpected:\n%v\n", indexPatterns, tc.expect)
Expand Down
Loading

0 comments on commit c52c193

Please sign in to comment.