diff --git a/generators/cmd/devstats/main.go b/generators/cmd/devstats/main.go new file mode 100644 index 00000000..78c8b121 --- /dev/null +++ b/generators/cmd/devstats/main.go @@ -0,0 +1,138 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package main + +import ( + "bytes" + _ "embed" + "flag" + "fmt" + "kubevirt.io/community/pkg/sigs" + "log" + "os" + "regexp" + "sort" + "text/template" +) + +type options struct { + sigsYAMLPath string + outputPath string +} + +func (o *options) Validate() error { + if o.sigsYAMLPath == "" { + return fmt.Errorf("path to sigs.yaml is required") + } + if _, err := os.Stat(o.sigsYAMLPath); os.IsNotExist(err) { + return fmt.Errorf("file %s does not exist", o.sigsYAMLPath) + } + return nil +} + +func gatherOptions() options { + o := options{} + fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + fs.StringVar(&o.sigsYAMLPath, "sigs-yaml-path", "./sigs.yaml", "path to file sigs.yaml") + fs.StringVar(&o.outputPath, "output-path", "/tmp/repo_groups.sql", "path to file to write the output into") + err := fs.Parse(os.Args[1:]) + if err != nil { + log.Fatalf("error parsing arguments %v: %v", os.Args[1:], err) + } + return o +} + +var repoNameMatcher = regexp.MustCompile(`^https://raw.githubusercontent.com/([^/]+/[^/]+)/.*$`) + +func main() { + opts := gatherOptions() + if err := opts.Validate(); err != nil { + log.Fatalf("invalid arguments: %v", err) + } + + sigsYAML, err := sigs.ReadFile(opts.sigsYAMLPath) + if err != nil { + log.Fatalf("failed to read sigs.yaml: %v", err) + } + + var d RepoGroupsTemplateData + for _, sig := range sigsYAML.Sigs { + repoGroup := RepoGroup{ + Name: sig.Name, + Alias: sig.Dir, + } + repoMap := make(map[string]struct{}) + for _, subProject := range sig.SubProjects { + for _, ownerRef := range subProject.Owners { + stringSubmatch := repoNameMatcher.FindStringSubmatch(ownerRef) + if stringSubmatch == nil { + log.Fatalf("ownerRef %q doesn't match!", ownerRef) + } + repoName := stringSubmatch[1] + if _, exists := repoMap[repoName]; !exists { + repoMap[repoName] = struct{}{} + } + } + } + if len(repoMap) == 0 { + continue + } + var repos []string + for repo := range repoMap { + repos = append(repos, repo) + } + sort.Strings(repos) + repoGroup.Repos = repos + d.RepoGroups = append(d.RepoGroups, repoGroup) + } + + sql, err := generateRepoGroupsSQL(d) + if err != nil { + log.Fatal(fmt.Errorf("failed to generate sql: %w", err)) + } + + file, err := os.OpenFile(opts.outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) + if err != nil { + log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err)) + } + defer file.Close() + _, err = file.WriteString(sql) + if err != nil { + log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err)) + } + + log.Printf("output written to %q", opts.outputPath) +} + +//go:embed repo_groups.gosql +var repoGroupsSQLTemplate string + +func generateRepoGroupsSQL(d RepoGroupsTemplateData) (string, error) { + templateInstance, err := template.New("repo_groups").Parse(repoGroupsSQLTemplate) + if err != nil { + return "", err + } + var output bytes.Buffer + err = templateInstance.Execute(&output, d) + if err != nil { + return "", err + } + return output.String(), nil +} diff --git a/generators/cmd/devstats/repo_groups.gosql b/generators/cmd/devstats/repo_groups.gosql new file mode 100644 index 00000000..d7bda869 --- /dev/null +++ b/generators/cmd/devstats/repo_groups.gosql @@ -0,0 +1,119 @@ +{{- /* + This file is part of the KubeVirt project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + limitations under the License. + + Copyright the KubeVirt Authors. + + */ +-}} +{{- /* gotype: kubevirt.io/community/generators/cmd/devstats.RepoGroupsTemplateData */ -}} +-- Add repository groups +with repo_latest as ( + select sub.repo_id, + sub.repo_name + from ( + select repo_id, + dup_repo_name as repo_name, + row_number() over (partition by repo_id order by created_at desc, id desc) as row_num + from + gha_events + ) sub + where + sub.row_num = 1 +) +update + gha_repos r +set + alias = ( + select rl.repo_name + from + repo_latest rl + where + rl.repo_id = r.id + ) +where + r.name like '%_/_%' + and r.name not like '%/%/%' +; + +-- Maybe some new repos specified in CTE config appeared, so their config will be changed +-- And because we only want to assign not specified to 'Other' - we do this configuration again +delete from gha_repo_groups; + +-- Per each SIG that has claimed ownership via one of it's subprojects we add a new entry in gha_repo_groups +-- 'repos' CTE is a full mapping between repo name N:M repo group +-- each line is ('repo/name', 'Repo Group Name'), +with repos as ( + select + repo, + repo_group + from ( + values{{ range $indexOuter, $repoGroup := $.RepoGroups }}{{ if $indexOuter }},{{ end }} + -- {{ $repoGroup.Name }}{{ range $indexInner, $repo := $repoGroup.Repos }}{{ if $indexInner }},{{ end }} + ('{{ $repo }}', '{{ $repoGroup.Name }}'){{ end }}{{ end }} + ) AS a (repo, repo_group) +) +insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login) +select + r.id, r.name, r.alias, c.repo_group, r.org_id, r.org_login +from + gha_repos r, + repos c +where + r.name = c.repo +; + +-- To see missing repos +/* +select + c.repo +from + repos c +left join + gha_repos r +on + r.name = c.repo +where + r.name is null; +; +*/ + + +-- Remaining repos that were not assigned to at least 1 repo group fall back to 'Other' repo group +insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login) +select + r.id, r.name, r.alias, 'Other', r.org_id, r.org_login +from + gha_repos r +left join + gha_repo_groups rg +on + r.name = rg.name +where + rg.name is null +; + +select + repo_group, + count(*) as number_of_repos +from + gha_repo_groups +where + repo_group is not null +group by + repo_group +order by + number_of_repos desc, + repo_group asc; \ No newline at end of file diff --git a/generators/cmd/devstats/repo_groups_test.go b/generators/cmd/devstats/repo_groups_test.go new file mode 100644 index 00000000..b947b4de --- /dev/null +++ b/generators/cmd/devstats/repo_groups_test.go @@ -0,0 +1,125 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package main + +import ( + "strings" + "testing" +) + +func TestRepoGroupsTemplate(t *testing.T) { + testCases := []struct { + name string + templateData RepoGroupsTemplateData + expectedOutputContained string + expectedErr error + }{ + { + name: "two groups", + templateData: RepoGroupsTemplateData{ + RepoGroups: []RepoGroup{ + { + Name: "sig-testing", + Alias: "blah", + Repos: []string{ + "kubevirt/kubevirt", + "kubevirt/test", + }, + }, + { + Name: "sig-ci", + Alias: "bled", + Repos: []string{ + "kubevirt/ci-health", + "kubevirt/kubevirtci", + }, + }, + }, + }, + expectedOutputContained: `from ( + values + -- sig-testing + ('kubevirt/kubevirt', 'sig-testing'), + ('kubevirt/test', 'sig-testing'), + -- sig-ci + ('kubevirt/ci-health', 'sig-ci'), + ('kubevirt/kubevirtci', 'sig-ci') + ) AS`, + expectedErr: nil, + }, + { + name: "three groups", + templateData: RepoGroupsTemplateData{ + RepoGroups: []RepoGroup{ + { + Name: "sig-testing", + Alias: "blah", + Repos: []string{ + "kubevirt/kubevirt", + "kubevirt/test", + }, + }, + { + Name: "sig-ci", + Alias: "bled", + Repos: []string{ + "kubevirt/ci-health", + "kubevirt/kubevirtci", + }, + }, + { + Name: "sig-buildsystem", + Alias: "bled", + Repos: []string{ + "kubevirt/kubevirt", + "kubevirt/project-infra", + }, + }, + }, + }, + expectedOutputContained: `from ( + values + -- sig-testing + ('kubevirt/kubevirt', 'sig-testing'), + ('kubevirt/test', 'sig-testing'), + -- sig-ci + ('kubevirt/ci-health', 'sig-ci'), + ('kubevirt/kubevirtci', 'sig-ci'), + -- sig-buildsystem + ('kubevirt/kubevirt', 'sig-buildsystem'), + ('kubevirt/project-infra', 'sig-buildsystem') + ) AS`, + expectedErr: nil, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + sql, err := generateRepoGroupsSQL(testCase.templateData) + if !strings.Contains(sql, testCase.expectedOutputContained) { + t.Log(sql) + t.Errorf(`wanted output to contain: +%s`, testCase.expectedOutputContained) + } + if testCase.expectedErr != err { + t.Errorf("got %q, want %q", err, testCase.expectedErr) + } + }) + } +} diff --git a/generators/cmd/devstats/types.go b/generators/cmd/devstats/types.go new file mode 100644 index 00000000..c8977103 --- /dev/null +++ b/generators/cmd/devstats/types.go @@ -0,0 +1,30 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + * + */ + +package main + +type RepoGroup struct { + Name string + Alias string + Repos []string +} + +type RepoGroupsTemplateData struct { + RepoGroups []RepoGroup +}