Skip to content

Commit

Permalink
devstats,repo_groups: generate repo_groups.sql
Browse files Browse the repository at this point in the history
Signed-off-by: Daniel Hiller <[email protected]>
  • Loading branch information
dhiller committed Oct 21, 2024
1 parent 76b09b5 commit a7e1bac
Showing 4 changed files with 412 additions and 0 deletions.
138 changes: 138 additions & 0 deletions generators/cmd/devstats/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright the KubeVirt Authors.
*
*/

package main

import (
"bytes"
_ "embed"
"flag"
"fmt"
"kubevirt.io/community/pkg/sigs"
"log"
"os"
"regexp"
"sort"
"text/template"
)

type options struct {
sigsYAMLPath string
outputPath string
}

func (o *options) Validate() error {
if o.sigsYAMLPath == "" {
return fmt.Errorf("path to sigs.yaml is required")
}
if _, err := os.Stat(o.sigsYAMLPath); os.IsNotExist(err) {
return fmt.Errorf("file %s does not exist", o.sigsYAMLPath)
}
return nil
}

func gatherOptions() options {
o := options{}
fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
fs.StringVar(&o.sigsYAMLPath, "sigs-yaml-path", "./sigs.yaml", "path to file sigs.yaml")
fs.StringVar(&o.outputPath, "output-path", "/tmp/repo_groups.sql", "path to file to write the output into")
err := fs.Parse(os.Args[1:])
if err != nil {
log.Fatalf("error parsing arguments %v: %v", os.Args[1:], err)
}
return o
}

var repoNameMatcher = regexp.MustCompile(`^https://raw.githubusercontent.com/([^/]+/[^/]+)/.*$`)

func main() {
opts := gatherOptions()
if err := opts.Validate(); err != nil {
log.Fatalf("invalid arguments: %v", err)
}

sigsYAML, err := sigs.ReadFile(opts.sigsYAMLPath)
if err != nil {
log.Fatalf("failed to read sigs.yaml: %v", err)
}

var d RepoGroupsTemplateData
for _, sig := range sigsYAML.Sigs {
repoGroup := RepoGroup{
Name: sig.Name,
Alias: sig.Dir,
}
repoMap := make(map[string]struct{})
for _, subProject := range sig.SubProjects {
for _, ownerRef := range subProject.Owners {
stringSubmatch := repoNameMatcher.FindStringSubmatch(ownerRef)
if stringSubmatch == nil {
log.Fatalf("ownerRef %q doesn't match!", ownerRef)
}
repoName := stringSubmatch[1]
if _, exists := repoMap[repoName]; !exists {
repoMap[repoName] = struct{}{}
}
}
}
if len(repoMap) == 0 {
continue
}
var repos []string
for repo := range repoMap {
repos = append(repos, repo)
}
sort.Strings(repos)
repoGroup.Repos = repos
d.RepoGroups = append(d.RepoGroups, repoGroup)
}

sql, err := generateRepoGroupsSQL(d)
if err != nil {
log.Fatal(fmt.Errorf("failed to generate sql: %w", err))
}

file, err := os.OpenFile(opts.outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err))
}
defer file.Close()
_, err = file.WriteString(sql)
if err != nil {
log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err))
}

log.Printf("output written to %q", opts.outputPath)
}

//go:embed repo_groups.gosql
var repoGroupsSQLTemplate string

func generateRepoGroupsSQL(d RepoGroupsTemplateData) (string, error) {
templateInstance, err := template.New("repo_groups").Parse(repoGroupsSQLTemplate)
if err != nil {
return "", err
}
var output bytes.Buffer
err = templateInstance.Execute(&output, d)
if err != nil {
return "", err
}
return output.String(), nil
}
119 changes: 119 additions & 0 deletions generators/cmd/devstats/repo_groups.gosql
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
{{- /*
This file is part of the KubeVirt project

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the License for the specific language governing permissions and
limitations under the License.

Copyright the KubeVirt Authors.

*/
-}}
{{- /* gotype: kubevirt.io/community/generators/cmd/devstats.RepoGroupsTemplateData */ -}}
-- Add repository groups
with repo_latest as (
select sub.repo_id,
sub.repo_name
from (
select repo_id,
dup_repo_name as repo_name,
row_number() over (partition by repo_id order by created_at desc, id desc) as row_num
from
gha_events
) sub
where
sub.row_num = 1
)
update
gha_repos r
set
alias = (
select rl.repo_name
from
repo_latest rl
where
rl.repo_id = r.id
)
where
r.name like '%_/_%'
and r.name not like '%/%/%'
;

-- Maybe some new repos specified in CTE config appeared, so their config will be changed
-- And because we only want to assign not specified to 'Other' - we do this configuration again
delete from gha_repo_groups;

-- Per each SIG that has claimed ownership via one of it's subprojects we add a new entry in gha_repo_groups
-- 'repos' CTE is a full mapping between repo name N:M repo group
-- each line is ('repo/name', 'Repo Group Name'),
with repos as (
select
repo,
repo_group
from (
values{{ range $indexOuter, $repoGroup := $.RepoGroups }}{{ if $indexOuter }},{{ end }}
-- {{ $repoGroup.Name }}{{ range $indexInner, $repo := $repoGroup.Repos }}{{ if $indexInner }},{{ end }}
('{{ $repo }}', '{{ $repoGroup.Name }}'){{ end }}{{ end }}
) AS a (repo, repo_group)
)
insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login)
select
r.id, r.name, r.alias, c.repo_group, r.org_id, r.org_login
from
gha_repos r,
repos c
where
r.name = c.repo
;

-- To see missing repos
/*
select
c.repo
from
repos c
left join
gha_repos r
on
r.name = c.repo
where
r.name is null;
;
*/


-- Remaining repos that were not assigned to at least 1 repo group fall back to 'Other' repo group
insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login)
select
r.id, r.name, r.alias, 'Other', r.org_id, r.org_login
from
gha_repos r
left join
gha_repo_groups rg
on
r.name = rg.name
where
rg.name is null
;

select
repo_group,
count(*) as number_of_repos
from
gha_repo_groups
where
repo_group is not null
group by
repo_group
order by
number_of_repos desc,
repo_group asc;
125 changes: 125 additions & 0 deletions generators/cmd/devstats/repo_groups_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright the KubeVirt Authors.
*
*/

package main

import (
"strings"
"testing"
)

func TestRepoGroupsTemplate(t *testing.T) {
testCases := []struct {
name string
templateData RepoGroupsTemplateData
expectedOutputContained string
expectedErr error
}{
{
name: "two groups",
templateData: RepoGroupsTemplateData{
RepoGroups: []RepoGroup{
{
Name: "sig-testing",
Alias: "blah",
Repos: []string{
"kubevirt/kubevirt",
"kubevirt/test",
},
},
{
Name: "sig-ci",
Alias: "bled",
Repos: []string{
"kubevirt/ci-health",
"kubevirt/kubevirtci",
},
},
},
},
expectedOutputContained: `from (
values
-- sig-testing
('kubevirt/kubevirt', 'sig-testing'),
('kubevirt/test', 'sig-testing'),
-- sig-ci
('kubevirt/ci-health', 'sig-ci'),
('kubevirt/kubevirtci', 'sig-ci')
) AS`,
expectedErr: nil,
},
{
name: "three groups",
templateData: RepoGroupsTemplateData{
RepoGroups: []RepoGroup{
{
Name: "sig-testing",
Alias: "blah",
Repos: []string{
"kubevirt/kubevirt",
"kubevirt/test",
},
},
{
Name: "sig-ci",
Alias: "bled",
Repos: []string{
"kubevirt/ci-health",
"kubevirt/kubevirtci",
},
},
{
Name: "sig-buildsystem",
Alias: "bled",
Repos: []string{
"kubevirt/kubevirt",
"kubevirt/project-infra",
},
},
},
},
expectedOutputContained: `from (
values
-- sig-testing
('kubevirt/kubevirt', 'sig-testing'),
('kubevirt/test', 'sig-testing'),
-- sig-ci
('kubevirt/ci-health', 'sig-ci'),
('kubevirt/kubevirtci', 'sig-ci'),
-- sig-buildsystem
('kubevirt/kubevirt', 'sig-buildsystem'),
('kubevirt/project-infra', 'sig-buildsystem')
) AS`,
expectedErr: nil,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
sql, err := generateRepoGroupsSQL(testCase.templateData)
if !strings.Contains(sql, testCase.expectedOutputContained) {
t.Log(sql)
t.Errorf(`wanted output to contain:
%s`, testCase.expectedOutputContained)
}
if testCase.expectedErr != err {
t.Errorf("got %q, want %q", err, testCase.expectedErr)
}
})
}
}
Loading

0 comments on commit a7e1bac

Please sign in to comment.