-
Notifications
You must be signed in to change notification settings - Fork 105
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
devstats,repo_groups: generate repo_groups.sql
Signed-off-by: Daniel Hiller <[email protected]>
- Loading branch information
Showing
4 changed files
with
412 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
/* | ||
* This file is part of the KubeVirt project | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
* | ||
* Copyright the KubeVirt Authors. | ||
* | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"bytes" | ||
_ "embed" | ||
"flag" | ||
"fmt" | ||
"kubevirt.io/community/pkg/sigs" | ||
"log" | ||
"os" | ||
"regexp" | ||
"sort" | ||
"text/template" | ||
) | ||
|
||
type options struct { | ||
sigsYAMLPath string | ||
outputPath string | ||
} | ||
|
||
func (o *options) Validate() error { | ||
if o.sigsYAMLPath == "" { | ||
return fmt.Errorf("path to sigs.yaml is required") | ||
} | ||
if _, err := os.Stat(o.sigsYAMLPath); os.IsNotExist(err) { | ||
return fmt.Errorf("file %s does not exist", o.sigsYAMLPath) | ||
} | ||
return nil | ||
} | ||
|
||
func gatherOptions() options { | ||
o := options{} | ||
fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) | ||
fs.StringVar(&o.sigsYAMLPath, "sigs-yaml-path", "./sigs.yaml", "path to file sigs.yaml") | ||
fs.StringVar(&o.outputPath, "output-path", "/tmp/repo_groups.sql", "path to file to write the output into") | ||
err := fs.Parse(os.Args[1:]) | ||
if err != nil { | ||
log.Fatalf("error parsing arguments %v: %v", os.Args[1:], err) | ||
} | ||
return o | ||
} | ||
|
||
var repoNameMatcher = regexp.MustCompile(`^https://raw.githubusercontent.com/([^/]+/[^/]+)/.*$`) | ||
|
||
func main() { | ||
opts := gatherOptions() | ||
if err := opts.Validate(); err != nil { | ||
log.Fatalf("invalid arguments: %v", err) | ||
} | ||
|
||
sigsYAML, err := sigs.ReadFile(opts.sigsYAMLPath) | ||
if err != nil { | ||
log.Fatalf("failed to read sigs.yaml: %v", err) | ||
} | ||
|
||
var d RepoGroupsTemplateData | ||
for _, sig := range sigsYAML.Sigs { | ||
repoGroup := RepoGroup{ | ||
Name: sig.Name, | ||
Alias: sig.Dir, | ||
} | ||
repoMap := make(map[string]struct{}) | ||
for _, subProject := range sig.SubProjects { | ||
for _, ownerRef := range subProject.Owners { | ||
stringSubmatch := repoNameMatcher.FindStringSubmatch(ownerRef) | ||
if stringSubmatch == nil { | ||
log.Fatalf("ownerRef %q doesn't match!", ownerRef) | ||
} | ||
repoName := stringSubmatch[1] | ||
if _, exists := repoMap[repoName]; !exists { | ||
repoMap[repoName] = struct{}{} | ||
} | ||
} | ||
} | ||
if len(repoMap) == 0 { | ||
continue | ||
} | ||
var repos []string | ||
for repo := range repoMap { | ||
repos = append(repos, repo) | ||
} | ||
sort.Strings(repos) | ||
repoGroup.Repos = repos | ||
d.RepoGroups = append(d.RepoGroups, repoGroup) | ||
} | ||
|
||
sql, err := generateRepoGroupsSQL(d) | ||
if err != nil { | ||
log.Fatal(fmt.Errorf("failed to generate sql: %w", err)) | ||
} | ||
|
||
file, err := os.OpenFile(opts.outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) | ||
if err != nil { | ||
log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err)) | ||
} | ||
defer file.Close() | ||
_, err = file.WriteString(sql) | ||
if err != nil { | ||
log.Fatal(fmt.Errorf("failed to write to file %q, %w", opts.outputPath, err)) | ||
} | ||
|
||
log.Printf("output written to %q", opts.outputPath) | ||
} | ||
|
||
//go:embed repo_groups.gosql | ||
var repoGroupsSQLTemplate string | ||
|
||
func generateRepoGroupsSQL(d RepoGroupsTemplateData) (string, error) { | ||
templateInstance, err := template.New("repo_groups").Parse(repoGroupsSQLTemplate) | ||
if err != nil { | ||
return "", err | ||
} | ||
var output bytes.Buffer | ||
err = templateInstance.Execute(&output, d) | ||
if err != nil { | ||
return "", err | ||
} | ||
return output.String(), nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
{{- /* | ||
This file is part of the KubeVirt project | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
|
||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
|
||
Copyright the KubeVirt Authors. | ||
|
||
*/ | ||
-}} | ||
{{- /* gotype: kubevirt.io/community/generators/cmd/devstats.RepoGroupsTemplateData */ -}} | ||
-- Add repository groups | ||
with repo_latest as ( | ||
select sub.repo_id, | ||
sub.repo_name | ||
from ( | ||
select repo_id, | ||
dup_repo_name as repo_name, | ||
row_number() over (partition by repo_id order by created_at desc, id desc) as row_num | ||
from | ||
gha_events | ||
) sub | ||
where | ||
sub.row_num = 1 | ||
) | ||
update | ||
gha_repos r | ||
set | ||
alias = ( | ||
select rl.repo_name | ||
from | ||
repo_latest rl | ||
where | ||
rl.repo_id = r.id | ||
) | ||
where | ||
r.name like '%_/_%' | ||
and r.name not like '%/%/%' | ||
; | ||
|
||
-- Maybe some new repos specified in CTE config appeared, so their config will be changed | ||
-- And because we only want to assign not specified to 'Other' - we do this configuration again | ||
delete from gha_repo_groups; | ||
|
||
-- Per each SIG that has claimed ownership via one of it's subprojects we add a new entry in gha_repo_groups | ||
-- 'repos' CTE is a full mapping between repo name N:M repo group | ||
-- each line is ('repo/name', 'Repo Group Name'), | ||
with repos as ( | ||
select | ||
repo, | ||
repo_group | ||
from ( | ||
values{{ range $indexOuter, $repoGroup := $.RepoGroups }}{{ if $indexOuter }},{{ end }} | ||
-- {{ $repoGroup.Name }}{{ range $indexInner, $repo := $repoGroup.Repos }}{{ if $indexInner }},{{ end }} | ||
('{{ $repo }}', '{{ $repoGroup.Name }}'){{ end }}{{ end }} | ||
) AS a (repo, repo_group) | ||
) | ||
insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login) | ||
select | ||
r.id, r.name, r.alias, c.repo_group, r.org_id, r.org_login | ||
from | ||
gha_repos r, | ||
repos c | ||
where | ||
r.name = c.repo | ||
; | ||
|
||
-- To see missing repos | ||
/* | ||
select | ||
c.repo | ||
from | ||
repos c | ||
left join | ||
gha_repos r | ||
on | ||
r.name = c.repo | ||
where | ||
r.name is null; | ||
; | ||
*/ | ||
|
||
|
||
-- Remaining repos that were not assigned to at least 1 repo group fall back to 'Other' repo group | ||
insert into gha_repo_groups(id, name, alias, repo_group, org_id, org_login) | ||
select | ||
r.id, r.name, r.alias, 'Other', r.org_id, r.org_login | ||
from | ||
gha_repos r | ||
left join | ||
gha_repo_groups rg | ||
on | ||
r.name = rg.name | ||
where | ||
rg.name is null | ||
; | ||
|
||
select | ||
repo_group, | ||
count(*) as number_of_repos | ||
from | ||
gha_repo_groups | ||
where | ||
repo_group is not null | ||
group by | ||
repo_group | ||
order by | ||
number_of_repos desc, | ||
repo_group asc; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
/* | ||
* This file is part of the KubeVirt project | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
* | ||
* Copyright the KubeVirt Authors. | ||
* | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"strings" | ||
"testing" | ||
) | ||
|
||
func TestRepoGroupsTemplate(t *testing.T) { | ||
testCases := []struct { | ||
name string | ||
templateData RepoGroupsTemplateData | ||
expectedOutputContained string | ||
expectedErr error | ||
}{ | ||
{ | ||
name: "two groups", | ||
templateData: RepoGroupsTemplateData{ | ||
RepoGroups: []RepoGroup{ | ||
{ | ||
Name: "sig-testing", | ||
Alias: "blah", | ||
Repos: []string{ | ||
"kubevirt/kubevirt", | ||
"kubevirt/test", | ||
}, | ||
}, | ||
{ | ||
Name: "sig-ci", | ||
Alias: "bled", | ||
Repos: []string{ | ||
"kubevirt/ci-health", | ||
"kubevirt/kubevirtci", | ||
}, | ||
}, | ||
}, | ||
}, | ||
expectedOutputContained: `from ( | ||
values | ||
-- sig-testing | ||
('kubevirt/kubevirt', 'sig-testing'), | ||
('kubevirt/test', 'sig-testing'), | ||
-- sig-ci | ||
('kubevirt/ci-health', 'sig-ci'), | ||
('kubevirt/kubevirtci', 'sig-ci') | ||
) AS`, | ||
expectedErr: nil, | ||
}, | ||
{ | ||
name: "three groups", | ||
templateData: RepoGroupsTemplateData{ | ||
RepoGroups: []RepoGroup{ | ||
{ | ||
Name: "sig-testing", | ||
Alias: "blah", | ||
Repos: []string{ | ||
"kubevirt/kubevirt", | ||
"kubevirt/test", | ||
}, | ||
}, | ||
{ | ||
Name: "sig-ci", | ||
Alias: "bled", | ||
Repos: []string{ | ||
"kubevirt/ci-health", | ||
"kubevirt/kubevirtci", | ||
}, | ||
}, | ||
{ | ||
Name: "sig-buildsystem", | ||
Alias: "bled", | ||
Repos: []string{ | ||
"kubevirt/kubevirt", | ||
"kubevirt/project-infra", | ||
}, | ||
}, | ||
}, | ||
}, | ||
expectedOutputContained: `from ( | ||
values | ||
-- sig-testing | ||
('kubevirt/kubevirt', 'sig-testing'), | ||
('kubevirt/test', 'sig-testing'), | ||
-- sig-ci | ||
('kubevirt/ci-health', 'sig-ci'), | ||
('kubevirt/kubevirtci', 'sig-ci'), | ||
-- sig-buildsystem | ||
('kubevirt/kubevirt', 'sig-buildsystem'), | ||
('kubevirt/project-infra', 'sig-buildsystem') | ||
) AS`, | ||
expectedErr: nil, | ||
}, | ||
} | ||
for _, testCase := range testCases { | ||
t.Run(testCase.name, func(t *testing.T) { | ||
sql, err := generateRepoGroupsSQL(testCase.templateData) | ||
if !strings.Contains(sql, testCase.expectedOutputContained) { | ||
t.Log(sql) | ||
t.Errorf(`wanted output to contain: | ||
%s`, testCase.expectedOutputContained) | ||
} | ||
if testCase.expectedErr != err { | ||
t.Errorf("got %q, want %q", err, testCase.expectedErr) | ||
} | ||
}) | ||
} | ||
} |
Oops, something went wrong.