devstats,repo_groups: generate repo_groups.sql (#332)

Signed-off-by: Daniel Hiller <[email protected]>

generators/cmd/devstats/main.go

@@ -0,0 +1,138 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright the KubeVirt Authors.
+ *
+ */
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"flag"
+	"fmt"
+	"kubevirt.io/community/pkg/sigs"
+	"log"
+	"os"
+	"regexp"
+	"sort"
+	"text/template"
+)
+
+type options struct {
+	sigsYAMLPath string
+	outputPath   string
+}
+
+func (o *options) Validate() error {
+	if o.sigsYAMLPath == "" {
+		return fmt.Errorf("path to sigs.yaml is required")
+	}
+	if _, err := os.Stat(o.sigsYAMLPath); os.IsNotExist(err) {
+		return fmt.Errorf("file %s does not exist", o.sigsYAMLPath)
+	}
+	return nil
+}
+
+func gatherOptions() options {
+	o := options{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+	fs.StringVar(&o.sigsYAMLPath, "sigs-yaml-path", "./sigs.yaml", "path to file sigs.yaml")
+	fs.StringVar(&o.outputPath, "output-path", "/tmp/repo_groups.sql", "path to file to write the output into")
+	err := fs.Parse(os.Args[1:])
+	if err != nil {
+		log.Fatalf("error parsing arguments %v: %v", os.Args[1:], err)
+	}
+	return o
+}
+
+var repoNameMatcher = regexp.MustCompile(`^https://raw.githubusercontent.com/([^/]+/[^/]+)/.*$`)
+
+func main() {
+	opts := gatherOptions()
+	if err := opts.Validate(); err != nil {
+		log.Fatalf("invalid arguments: %v", err)
+	}
+
+	sigsYAML, err := sigs.ReadFile(opts.sigsYAMLPath)
+	if err != nil {
+		log.Fatalf("failed to read sigs.yaml: %v", err)
+	}
+
+	var d RepoGroupsTemplateData
+	for _, sig := range sigsYAML.Sigs {
+		repoGroup := RepoGroup{
+			Name:  sig.Name,
+			Alias: sig.Dir,
+		}
+		repoMap := make(map[string]struct{})
+		for _, subProject := range sig.SubProjects {
+			for _, ownerRef := range subProject.Owners {
+				stringSubmatch := repoNameMatcher.FindStringSubmatch(ownerRef)
+				if stringSubmatch == nil {
+					log.Fatalf("ownerRef %q doesn't match!", ownerRef)
+				}
+				repoName := stringSubmatch[1]
+				if _, exists := repoMap[repoName]; !exists {
+					repoMap[repoName] = struct{}{}
+				}
+			}
+		}
+		if len(repoMap) == 0 {
+			continue
+		}
+		var repos []string
+		for repo := range repoMap {
+			repos = append(repos, repo)
+		}
+		sort.Strings(repos)
+		repoGroup.Repos = repos
+		d.RepoGroups = append(d.RepoGroups, repoGroup)
+	}
+
+	sql, err := generateRepoGroupsSQL(d)
+	if err != nil {
+		log.Fatal(fmt.Errorf("failed to generate sql: %w", err))
+	}
+
+	file, err := os.OpenFile(opts.outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err))
+	}
+	defer file.Close()
+	_, err = file.WriteString(sql)
+	if err != nil {
+		log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err))
+	}
+
+	log.Printf("output written to %q", opts.outputPath)
+}
+
+//go:embed repo_groups.gosql
+var repoGroupsSQLTemplate string
+
+func generateRepoGroupsSQL(d RepoGroupsTemplateData) (string, error) {
+	templateInstance, err := template.New("repo_groups").Parse(repoGroupsSQLTemplate)
+	if err != nil {
+		return "", err
+	}
+	var output bytes.Buffer
+	err = templateInstance.Execute(&output, d)
+	if err != nil {
+		return "", err
+	}
+	return output.String(), nil
+}

generators/cmd/devstats/repo_groups.gosql

@@ -0,0 +1,119 @@
+{{- /*
+    This file is part of the KubeVirt project
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+    Copyright the KubeVirt Authors.
+
+    */
+-}}
+{{- /* gotype: kubevirt.io/community/generators/cmd/devstats.RepoGroupsTemplateData */ -}}
+-- Add repository groups
+with repo_latest as (
+  select sub.repo_id,
+    sub.repo_name
+  from (
+    select repo_id,
+      dup_repo_name as repo_name,
+      row_number() over (partition by repo_id order by created_at desc, id desc) as row_num
+    from
+      gha_events
+  ) sub
+  where
+    sub.row_num = 1
+)
+update
+  gha_repos r
+set
+  alias = (
+    select rl.repo_name
+    from
+      repo_latest rl
+    where
+      rl.repo_id = r.id
+  )
+where
+  r.name like '%_/_%'
+  and r.name not like '%/%/%'
+;
+
+-- Maybe some new repos specified in CTE config appeared, so their config will be changed
+-- And because we only want to assign not specified to 'Other' - we do this configuration again
+delete from gha_repo_groups;
+
+-- Per each SIG that has claimed ownership via one of it's subprojects we add a new entry in gha_repo_groups
+-- 'repos' CTE is a full mapping between repo name N:M repo group
+-- each line is ('repo/name', 'Repo Group Name'),
+with repos as (
+  select
+    repo,
+    repo_group
+  from (
+    values{{ range $indexOuter, $repoGroup := $.RepoGroups }}{{ if $indexOuter }},{{ end }}
+      -- {{ $repoGroup.Name }}{{ range $indexInner, $repo := $repoGroup.Repos }}{{ if $indexInner }},{{ end }}
+      ('{{ $repo }}', '{{ $repoGroup.Name }}'){{ end }}{{ end }}
+  ) AS a (repo, repo_group)
+)
+insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login)
+select
+  r.id, r.name, r.alias, c.repo_group, r.org_id, r.org_login
+from
+  gha_repos r,
+  repos c
+where
+  r.name = c.repo
+;
+
+-- To see missing repos
+/*
+select
+  c.repo
+from
+  repos c
+left join
+  gha_repos r
+on
+  r.name = c.repo
+where
+  r.name is null;
+;
+*/
+
+
+-- Remaining repos that were not assigned to at least 1 repo group fall back to 'Other' repo group
+insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login)
+select
+  r.id, r.name, r.alias, 'Other', r.org_id, r.org_login
+from
+  gha_repos r
+left join
+  gha_repo_groups rg
+on
+  r.name = rg.name
+where
+  rg.name is null
+;
+
+select
+  repo_group,
+  count(*) as number_of_repos
+from
+  gha_repo_groups
+where
+  repo_group is not null
+group by
+  repo_group
+order by
+  number_of_repos desc,
+  repo_group asc;

generators/cmd/devstats/repo_groups_test.go

@@ -0,0 +1,125 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright the KubeVirt Authors.
+ *
+ */
+
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRepoGroupsTemplate(t *testing.T) {
+	testCases := []struct {
+		name                    string
+		templateData            RepoGroupsTemplateData
+		expectedOutputContained string
+		expectedErr             error
+	}{
+		{
+			name: "two groups",
+			templateData: RepoGroupsTemplateData{
+				RepoGroups: []RepoGroup{
+					{
+						Name:  "sig-testing",
+						Alias: "blah",
+						Repos: []string{
+							"kubevirt/kubevirt",
+							"kubevirt/test",
+						},
+					},
+					{
+						Name:  "sig-ci",
+						Alias: "bled",
+						Repos: []string{
+							"kubevirt/ci-health",
+							"kubevirt/kubevirtci",
+						},
+					},
+				},
+			},
+			expectedOutputContained: `from (
+    values
+      -- sig-testing
+      ('kubevirt/kubevirt', 'sig-testing'),
+      ('kubevirt/test', 'sig-testing'),
+      -- sig-ci
+      ('kubevirt/ci-health', 'sig-ci'),
+      ('kubevirt/kubevirtci', 'sig-ci')
+  ) AS`,
+			expectedErr: nil,
+		},
+		{
+			name: "three groups",
+			templateData: RepoGroupsTemplateData{
+				RepoGroups: []RepoGroup{
+					{
+						Name:  "sig-testing",
+						Alias: "blah",
+						Repos: []string{
+							"kubevirt/kubevirt",
+							"kubevirt/test",
+						},
+					},
+					{
+						Name:  "sig-ci",
+						Alias: "bled",
+						Repos: []string{
+							"kubevirt/ci-health",
+							"kubevirt/kubevirtci",
+						},
+					},
+					{
+						Name:  "sig-buildsystem",
+						Alias: "bled",
+						Repos: []string{
+							"kubevirt/kubevirt",
+							"kubevirt/project-infra",
+						},
+					},
+				},
+			},
+			expectedOutputContained: `from (
+    values
+      -- sig-testing
+      ('kubevirt/kubevirt', 'sig-testing'),
+      ('kubevirt/test', 'sig-testing'),
+      -- sig-ci
+      ('kubevirt/ci-health', 'sig-ci'),
+      ('kubevirt/kubevirtci', 'sig-ci'),
+      -- sig-buildsystem
+      ('kubevirt/kubevirt', 'sig-buildsystem'),
+      ('kubevirt/project-infra', 'sig-buildsystem')
+  ) AS`,
+			expectedErr: nil,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			sql, err := generateRepoGroupsSQL(testCase.templateData)
+			if !strings.Contains(sql, testCase.expectedOutputContained) {
+				t.Log(sql)
+				t.Errorf(`wanted output to contain:
+%s`, testCase.expectedOutputContained)
+			}
+			if testCase.expectedErr != err {
+				t.Errorf("got %q, want %q", err, testCase.expectedErr)
+			}
+		})
+	}
+}