From fd70c8fe562fedb5024fff3d23dc473e1a0fd873 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 11:07:37 -0700 Subject: [PATCH 01/24] Including Regular Expression checker when using Repository Search --- cmd/platform.go | 20 ++++--- internal/scm/github/github.go | 110 ++++++++++++++++++++++++++++++---- 2 files changed, 112 insertions(+), 18 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index a8018c4a..9181fc43 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -28,6 +28,8 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.") flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") + flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target.") + flags.StringP("filter-file", "", "", "Point to a file to filter out certain target repositories") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") flags.BoolP("include-subgroups", "", false, "Include GitLab subgroups when using the --group flag.") @@ -122,14 +124,16 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu orgs, _ := flag.GetStringSlice("org") users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") + repoSearch, _ := flag.GetString("repo-search") + filterFile, _ := flag.GetString("filter-file") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") sshAuth, _ := flag.GetBool("ssh-auth") skipForks, _ := flag.GetBool("skip-forks") - if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 { - return nil, errors.New("no organization, user or repo set") + if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" { + return nil, errors.New("no organization, user, repo or repo-search set") } token, err := getToken(flag) @@ -164,11 +168,13 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu BaseURL: gitBaseURL, TransportMiddleware: http.NewLoggingRoundTripper, RepoListing: github.RepositoryListing{ - Organizations: orgs, - Users: users, - Repositories: repoRefs, - Topics: topics, - SkipForks: skipForks, + Organizations: orgs, + Users: users, + Repositories: repoRefs, + RepositorySearch: repoSearch, + Topics: topics, + SkipForks: skipForks, + FilterFile: filterFile, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index a2391b06..583eb684 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -1,19 +1,22 @@ package github import ( + "bufio" "context" "fmt" - "net/http" - "sort" - "strings" - "sync" - "time" - "github.com/google/go-github/v55/github" "github.com/lindell/multi-gitter/internal/scm" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/oauth2" + "net/http" + "os" + "regexp" + "slices" + "sort" + "strings" + "sync" + "time" ) type Config struct { @@ -105,11 +108,13 @@ type Github struct { // RepositoryListing contains information about which repositories that should be fetched type RepositoryListing struct { - Organizations []string - Users []string - Repositories []RepositoryReference - Topics []string - SkipForks bool + Organizations []string + Users []string + Repositories []RepositoryReference + RepositorySearch string + Topics []string + SkipForks bool + FilterFile string } // RepositoryReference contains information to be able to reference a repository @@ -213,6 +218,13 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err allRepos = append(allRepos, repo) } + if len(g.RepositorySearch) > 0 { + repos, err := g.getSearchRepositories(ctx, g.RepositorySearch) + if err != nil { + return nil, errors.Wrapf(err, "could not get repository search results for '%s'", g.RepositorySearch) + } + allRepos = append(allRepos, repos...) + } // Remove duplicate repos repoMap := map[string]*github.Repository{} for _, repo := range allRepos { @@ -282,6 +294,82 @@ func (g *Github) getUserRepositories(ctx context.Context, user string) ([]*githu return repos, nil } +func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*github.Repository, error) { + var repos []*github.Repository + i := 1 + for { + rr, _, err := retry(ctx, func() ([]*github.Repository, *github.Response, error) { + rr, resp, err := g.ghClient.Search.Repositories(ctx, search, &github.SearchOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: 100, + }, + }) + + if err != nil { + return nil, nil, err + } + + if rr.IncompleteResults != nil && *rr.IncompleteResults { + // can occur when search times out on the server: for now, fail instead + // of handling the issue + return nil, nil, fmt.Errorf("search results incomplete") + } + + return rr.Repositories, resp, nil + }) + + if err != nil { + return nil, err + } + + repos = append(repos, rr...) + if len(rr) != 100 { + break + } + i++ + } + + /* + TODO: + * Generate Repositories that match name with Repository search | Done from @jamestelfer + * Verify returned repositories match repo name using regular expression, if not remove them from list + * In addition if a a ignore file exist, use that before returning the set of found repositories + */ + var ignoreList []string + if len(g.FilterFile) > 0 { + /// Go ahead and fill ignore_list with excluded repositories + file, openErr := os.Open(g.FilterFile) + defer func() { + err := file.Close() + if err != nil { + fmt.Printf("Error closing file: %s", err) + } + }() + if openErr != nil { + fmt.Printf("Error: %s", openErr) + } else { + FilterScanner := bufio.NewScanner(file) + for FilterScanner.Scan() { + line := FilterScanner.Text() + ignoreList = append(ignoreList, line) + } + scanError := FilterScanner.Err() + if scanError != nil { + fmt.Printf("Error reading file: %s", scanError) + } + } + } + for index := len(repos) - 1; index > 0; index-- { + matchPattern := fmt.Sprintf("^%s", search) + match, _ := regexp.MatchString(matchPattern, *repos[index].FullName) + if !match || slices.Contains(ignoreList, *repos[index].FullName) { + repos = append(repos[:index], repos[index+1:]...) + } + } + return repos, nil +} + func (g *Github) getRepository(ctx context.Context, repoRef RepositoryReference) (*github.Repository, error) { repo, _, err := retry(ctx, func() (*github.Repository, *github.Response, error) { return g.ghClient.Repositories.Get(ctx, repoRef.OwnerName, repoRef.Name) From a87dec1e14a897198b60250ce8c3bc3354feb5a6 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 11:25:14 -0700 Subject: [PATCH 02/24] Removing unused comments --- internal/scm/github/github.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 583eb684..045873a1 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -329,16 +329,9 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g } i++ } - - /* - TODO: - * Generate Repositories that match name with Repository search | Done from @jamestelfer - * Verify returned repositories match repo name using regular expression, if not remove them from list - * In addition if a a ignore file exist, use that before returning the set of found repositories - */ var ignoreList []string if len(g.FilterFile) > 0 { - /// Go ahead and fill ignore_list with excluded repositories + //fill ignore_list with excluded repositories file, openErr := os.Open(g.FilterFile) defer func() { err := file.Close() From 5f08f6de960b33df43ce554cc6eb8aac922f4544 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 12:06:14 -0700 Subject: [PATCH 03/24] Including unit test --- internal/scm/github/github_test.go | 91 ++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index 38f65fab..6e467dab 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "os" "strings" "testing" @@ -120,6 +121,64 @@ func Test_GetRepositories(t *testing.T) { "created_at": "2020-01-03T16:49:16Z" } ]`, + "/search/repositories": `{ + "total_count": 1, + "incomplete_results": false, + "items": [ + { + "id": 3, + "name": "repo-1", + "full_name": "lindell/repo-1", + "private": false, + "topics": [ + "backend", + "go" + ], + "owner": { + "login": "lindell", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/lindell/repo-1", + "fork": true, + "archived": false, + "disabled": false, + "default_branch": "main", + "permissions": { + "admin": true, + "push": true, + "pull": true + }, + "created_at": "2020-01-03T16:49:16Z" + }, + { + "id": 3, + "name": "repo-1", + "full_name": "lindell/repo-2", + "private": false, + "topics": [ + "backend", + "go" + ], + "owner": { + "login": "lindell", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/lindell/repo-2", + "fork": true, + "archived": false, + "disabled": false, + "default_branch": "main", + "permissions": { + "admin": true, + "push": true, + "pull": true + }, + "created_at": "2020-01-03T16:49:16Z" + } + ] + }`, }, } @@ -166,6 +225,38 @@ func Test_GetRepositories(t *testing.T) { } } + // repository with filter-file + { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + RepositorySearch: "lindell/repo-", + FilterFile: "ignore.txt", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) + testFile, err := os.Create("ignore.txt") + assert.NoError(t, err) + ignore_repositories := []string{ + "lindell/repo-2", // Will remove lindell/repo-2 from repos slice + } + defer func() { + closeErr := testFile.Close() + assert.NoError(t, closeErr) + removeErr := os.Remove(testFile.Name()) + assert.NoError(t, removeErr) + }() + for _, line := range ignore_repositories { + _, writeErr := io.WriteString(testFile, line) + assert.NoError(t, writeErr) + } + repos, err := gh.GetRepositories(context.Background()) + assert.NoError(t, err) + assert.Equal(t, len(repos), 1) + + } + // User { gh, err := github.New(github.Config{ From e5b99af7c0654b6b839f2cf3286b455d711e3d3c Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 12:48:59 -0700 Subject: [PATCH 04/24] organizing imports --- internal/scm/github/github.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 045873a1..88d418f1 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -4,11 +4,6 @@ import ( "bufio" "context" "fmt" - "github.com/google/go-github/v55/github" - "github.com/lindell/multi-gitter/internal/scm" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "golang.org/x/oauth2" "net/http" "os" "regexp" @@ -17,6 +12,12 @@ import ( "strings" "sync" "time" + + "github.com/google/go-github/v55/github" + "github.com/lindell/multi-gitter/internal/scm" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" ) type Config struct { From ebacb0a41ac1c0441b3b5c3b5e3d54f5be53782a Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 14:58:26 -0700 Subject: [PATCH 05/24] Utilizing --repo-exclude to exclude certain repositories --- cmd/platform.go | 6 +- internal/scm/github/github.go | 95 ++++++++++++++++-------------- internal/scm/github/github_test.go | 48 +++++++++------ 3 files changed, 83 insertions(+), 66 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index 9181fc43..94b8e883 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -29,7 +29,7 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target.") - flags.StringP("filter-file", "", "", "Point to a file to filter out certain target repositories") + flags.StringP("repo-exclude", "", "", "Exclude certain repositories with a regex string") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") flags.BoolP("include-subgroups", "", false, "Include GitLab subgroups when using the --group flag.") @@ -125,7 +125,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") repoSearch, _ := flag.GetString("repo-search") - filterFile, _ := flag.GetString("filter-file") + repoExclude, _ := flag.GetString("repo-exclude") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") @@ -174,7 +174,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu RepositorySearch: repoSearch, Topics: topics, SkipForks: skipForks, - FilterFile: filterFile, + RepoExclude: repoExclude, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 88d418f1..5f8efd68 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -1,11 +1,11 @@ package github import ( - "bufio" + "cmp" "context" "fmt" + "golang.org/x/oauth2" "net/http" - "os" "regexp" "slices" "sort" @@ -17,7 +17,6 @@ import ( "github.com/lindell/multi-gitter/internal/scm" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "golang.org/x/oauth2" ) type Config struct { @@ -115,7 +114,7 @@ type RepositoryListing struct { RepositorySearch string Topics []string SkipForks bool - FilterFile string + RepoExclude string } // RepositoryReference contains information to be able to reference a repository @@ -227,21 +226,27 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err allRepos = append(allRepos, repos...) } // Remove duplicate repos - repoMap := map[string]*github.Repository{} - for _, repo := range allRepos { - repoMap[repo.GetFullName()] = repo - } - allRepos = make([]*github.Repository, 0, len(repoMap)) - for _, repo := range repoMap { - if repo.GetArchived() || repo.GetDisabled() { - continue + var state string // Will be used to detect duplicates + slices.SortFunc(allRepos, func(a, b *github.Repository) int { + return cmp.Compare(strings.ToLower(*a.FullName), strings.ToLower(*b.FullName)) + }) + for index := len(allRepos) - 1; index >= 0; index-- { + ignoreRepo := false + if g.RepoExclude != "" { // Exclude repositories if --repo-exclude is used + match, _ := regexp.MatchString(g.RepoExclude, *allRepos[index].FullName) + ignoreRepo = match + } + fmt.Printf("State: %s\n", state) + if allRepos[index].GetArchived() || allRepos[index].GetDisabled() || ignoreRepo || strings.ToLower(*allRepos[index].FullName) == state { + state = *allRepos[index].FullName + allRepos = append(allRepos[:index], allRepos[index+1:]...) // Remove element + } else { + state = strings.ToLower(*allRepos[index].FullName) } - allRepos = append(allRepos, repo) } sort.Slice(allRepos, func(i, j int) bool { return allRepos[i].GetCreatedAt().Before(allRepos[j].GetCreatedAt().Time) }) - return allRepos, nil } @@ -330,37 +335,37 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g } i++ } - var ignoreList []string - if len(g.FilterFile) > 0 { - //fill ignore_list with excluded repositories - file, openErr := os.Open(g.FilterFile) - defer func() { - err := file.Close() - if err != nil { - fmt.Printf("Error closing file: %s", err) - } - }() - if openErr != nil { - fmt.Printf("Error: %s", openErr) - } else { - FilterScanner := bufio.NewScanner(file) - for FilterScanner.Scan() { - line := FilterScanner.Text() - ignoreList = append(ignoreList, line) - } - scanError := FilterScanner.Err() - if scanError != nil { - fmt.Printf("Error reading file: %s", scanError) - } - } - } - for index := len(repos) - 1; index > 0; index-- { - matchPattern := fmt.Sprintf("^%s", search) - match, _ := regexp.MatchString(matchPattern, *repos[index].FullName) - if !match || slices.Contains(ignoreList, *repos[index].FullName) { - repos = append(repos[:index], repos[index+1:]...) - } - } + //var ignoreList []string + //if len(g.FilterFile) > 0 { + // //fill ignore_list with excluded repositories + // file, openErr := os.Open(g.FilterFile) + // defer func() { + // err := file.Close() + // if err != nil { + // fmt.Printf("Error closing file: %s", err) + // } + // }() + // if openErr != nil { + // fmt.Printf("Error: %s", openErr) + // } else { + // FilterScanner := bufio.NewScanner(file) + // for FilterScanner.Scan() { + // line := FilterScanner.Text() + // ignoreList = append(ignoreList, line) + // } + // scanError := FilterScanner.Err() + // if scanError != nil { + // fmt.Printf("Error reading file: %s", scanError) + // } + // } + //} + //for index := len(repos) - 1; index > 0; index-- { + // matchPattern := fmt.Sprintf("^%s", search) + // match, _ := regexp.MatchString(matchPattern, *repos[index].FullName) + // if !match || slices.Contains(ignoreList, *repos[index].FullName) { + // repos = append(repos[:index], repos[index+1:]...) + // } + //} return repos, nil } diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index 6e467dab..3a83edee 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -3,9 +3,9 @@ package github_test import ( "context" "errors" + "fmt" "io" "net/http" - "os" "strings" "testing" @@ -153,7 +153,7 @@ func Test_GetRepositories(t *testing.T) { }, { "id": 3, - "name": "repo-1", + "name": "repo-2", "full_name": "lindell/repo-2", "private": false, "topics": [ @@ -176,6 +176,32 @@ func Test_GetRepositories(t *testing.T) { "pull": true }, "created_at": "2020-01-03T16:49:16Z" + }, + { + "id": 4, + "name": "repo-1", + "full_name": "lindell/repo-1", + "private": false, + "topics": [ + "backend", + "go" + ], + "owner": { + "login": "lindell", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/lindell/repo-1", + "fork": true, + "archived": false, + "disabled": false, + "default_branch": "main", + "permissions": { + "admin": true, + "push": true, + "pull": true + }, + "created_at": "2020-01-03T16:49:16Z" } ] }`, @@ -231,28 +257,14 @@ func Test_GetRepositories(t *testing.T) { TransportMiddleware: transport.Wrapper, RepoListing: github.RepositoryListing{ RepositorySearch: "lindell/repo-", - FilterFile: "ignore.txt", + RepoExclude: "^lindell/repo-2$", }, MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, }) require.NoError(t, err) - testFile, err := os.Create("ignore.txt") - assert.NoError(t, err) - ignore_repositories := []string{ - "lindell/repo-2", // Will remove lindell/repo-2 from repos slice - } - defer func() { - closeErr := testFile.Close() - assert.NoError(t, closeErr) - removeErr := os.Remove(testFile.Name()) - assert.NoError(t, removeErr) - }() - for _, line := range ignore_repositories { - _, writeErr := io.WriteString(testFile, line) - assert.NoError(t, writeErr) - } repos, err := gh.GetRepositories(context.Background()) assert.NoError(t, err) + fmt.Println(repos) assert.Equal(t, len(repos), 1) } From 797f1ad2257b401905c35880cc4ff1616893d593 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 15:05:03 -0700 Subject: [PATCH 06/24] Removing unused comments --- internal/scm/github/github.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 5f8efd68..5d1d76fb 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -335,37 +335,6 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g } i++ } - //var ignoreList []string - //if len(g.FilterFile) > 0 { - // //fill ignore_list with excluded repositories - // file, openErr := os.Open(g.FilterFile) - // defer func() { - // err := file.Close() - // if err != nil { - // fmt.Printf("Error closing file: %s", err) - // } - // }() - // if openErr != nil { - // fmt.Printf("Error: %s", openErr) - // } else { - // FilterScanner := bufio.NewScanner(file) - // for FilterScanner.Scan() { - // line := FilterScanner.Text() - // ignoreList = append(ignoreList, line) - // } - // scanError := FilterScanner.Err() - // if scanError != nil { - // fmt.Printf("Error reading file: %s", scanError) - // } - // } - //} - //for index := len(repos) - 1; index > 0; index-- { - // matchPattern := fmt.Sprintf("^%s", search) - // match, _ := regexp.MatchString(matchPattern, *repos[index].FullName) - // if !match || slices.Contains(ignoreList, *repos[index].FullName) { - // repos = append(repos[:index], repos[index+1:]...) - // } - //} return repos, nil } From 3d3d49dced6a4c2e08195295061df6d816c038a5 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 25 Oct 2023 15:09:55 -0700 Subject: [PATCH 07/24] Removing extra comment --- internal/scm/github/github.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 5d1d76fb..7f003929 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -236,7 +236,6 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err match, _ := regexp.MatchString(g.RepoExclude, *allRepos[index].FullName) ignoreRepo = match } - fmt.Printf("State: %s\n", state) if allRepos[index].GetArchived() || allRepos[index].GetDisabled() || ignoreRepo || strings.ToLower(*allRepos[index].FullName) == state { state = *allRepos[index].FullName allRepos = append(allRepos[:index], allRepos[index+1:]...) // Remove element From 72f80136ff4902d58fc57f6d68418fa94468a72d Mon Sep 17 00:00:00 2001 From: josealdaco Date: Fri, 27 Oct 2023 11:01:51 -0700 Subject: [PATCH 08/24] Refactor based on PR feedback --- cmd/platform.go | 11 ++++----- internal/scm/github/github.go | 38 +++++++++++++++--------------- internal/scm/github/github_test.go | 4 ++-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index 94b8e883..2ffe2396 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -28,8 +28,8 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.") flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") - flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target.") - flags.StringP("repo-exclude", "", "", "Exclude certain repositories with a regex string") + flags.StringSliceP("repo-search", "", nil, "Use a repository search to find repositories to target.") + flags.StringP("repo-filter", "", "", "Filter repositories with a Regular expression") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") flags.BoolP("include-subgroups", "", false, "Include GitLab subgroups when using the --group flag.") @@ -125,14 +125,13 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") repoSearch, _ := flag.GetString("repo-search") - repoExclude, _ := flag.GetString("repo-exclude") + repoFilter, _ := flag.GetString("repo-filter") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") sshAuth, _ := flag.GetBool("ssh-auth") skipForks, _ := flag.GetBool("skip-forks") - - if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" { + if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && len(repoSearch) == 0 { return nil, errors.New("no organization, user, repo or repo-search set") } @@ -174,7 +173,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu RepositorySearch: repoSearch, Topics: topics, SkipForks: skipForks, - RepoExclude: repoExclude, + RepositoryFilter: repoFilter, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 7f003929..bd641d9e 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -114,7 +114,7 @@ type RepositoryListing struct { RepositorySearch string Topics []string SkipForks bool - RepoExclude string + RepositoryFilter string } // RepositoryReference contains information to be able to reference a repository @@ -193,7 +193,10 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error) func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) { allRepos := []*github.Repository{} - + var repoRegEx *regexp.Regexp // repoFilter flag will filter all collected repositories + if g.RepositoryFilter != "" { + repoRegEx = regexp.MustCompile(g.RepositoryFilter) + } for _, org := range g.Organizations { repos, err := g.getOrganizationRepositories(ctx, org) if err != nil { @@ -225,24 +228,21 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repos...) } - // Remove duplicate repos - var state string // Will be used to detect duplicates - slices.SortFunc(allRepos, func(a, b *github.Repository) int { - return cmp.Compare(strings.ToLower(*a.FullName), strings.ToLower(*b.FullName)) + filteredRepos := make([]*github.Repository, 0, len(allRepos)) + // Filter repositories + filteredRepos = slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { + regExMatch := repoRegEx != nil && repoRegEx.MatchString(repo.GetFullName()) + return regExMatch || (repo.GetArchived() || repo.GetDisabled()) }) - for index := len(allRepos) - 1; index >= 0; index-- { - ignoreRepo := false - if g.RepoExclude != "" { // Exclude repositories if --repo-exclude is used - match, _ := regexp.MatchString(g.RepoExclude, *allRepos[index].FullName) - ignoreRepo = match - } - if allRepos[index].GetArchived() || allRepos[index].GetDisabled() || ignoreRepo || strings.ToLower(*allRepos[index].FullName) == state { - state = *allRepos[index].FullName - allRepos = append(allRepos[:index], allRepos[index+1:]...) // Remove element - } else { - state = strings.ToLower(*allRepos[index].FullName) - } - } + // Remove duplicates + allRepos = make([]*github.Repository, 0, len(filteredRepos)) + slices.SortFunc(filteredRepos, func(a, b *github.Repository) int { + return cmp.Compare(a.GetFullName(), b.GetFullName()) + }) + allRepos = slices.CompactFunc(filteredRepos, func(g2 *github.Repository, g *github.Repository) bool { + return g2.GetFullName() == g.GetFullName() + }) + // Sort by Datetime sort.Slice(allRepos, func(i, j int) bool { return allRepos[i].GetCreatedAt().Before(allRepos[j].GetCreatedAt().Time) }) diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index 3a83edee..a8537776 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -257,7 +257,7 @@ func Test_GetRepositories(t *testing.T) { TransportMiddleware: transport.Wrapper, RepoListing: github.RepositoryListing{ RepositorySearch: "lindell/repo-", - RepoExclude: "^lindell/repo-2$", + RepositoryFilter: "^lindell/repo-2$", }, MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, }) @@ -265,7 +265,7 @@ func Test_GetRepositories(t *testing.T) { repos, err := gh.GetRepositories(context.Background()) assert.NoError(t, err) fmt.Println(repos) - assert.Equal(t, len(repos), 1) + assert.Equal(t, 1, len(repos)) } From b2ea5dd93c9c01a1b59f92066facdd83f099c275 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Fri, 27 Oct 2023 12:24:25 -0700 Subject: [PATCH 09/24] Minor id change on RepoFilter test --- internal/scm/github/github_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index efdb5bc9..ec133351 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -373,7 +373,7 @@ func Test_RepositoryFilter(t *testing.T) { "incomplete_results": false, "items": [ { - "id": 3, + "id": 1, "name": "search-repo1", "full_name": "lindell/search-repo1", "private": false, @@ -399,7 +399,7 @@ func Test_RepositoryFilter(t *testing.T) { "created_at": "2020-01-03T16:49:19Z" }, { - "id": 4, + "id": 2, "name": "search-repo-2", "full_name": "lindell/search-repo-2", "private": false, From 75832675711224a9c3220b1152d705baecb81a29 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Fri, 27 Oct 2023 13:10:48 -0700 Subject: [PATCH 10/24] Adding nil statement in case a regularexpression was not used --- internal/scm/github/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index b26f4133..d76385f0 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -232,7 +232,7 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err // Filter repositories filteredRepos = slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { regExMatch := repoRegEx != nil && repoRegEx.MatchString(repo.GetFullName()) - return regExMatch || (repo.GetArchived() || repo.GetDisabled()) + return (!regExMatch && repoRegEx != nil) || (repo.GetArchived() || repo.GetDisabled()) }) // Remove duplicates allRepos = make([]*github.Repository, 0, len(filteredRepos)) From 5909e8a34222463ec1c8b6571a058b78ed65fdc9 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Fri, 27 Oct 2023 13:47:40 -0700 Subject: [PATCH 11/24] fix package import structure --- internal/scm/github/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index d76385f0..be87eae6 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "fmt" - "golang.org/x/oauth2" "net/http" "regexp" "slices" @@ -17,6 +16,7 @@ import ( "github.com/lindell/multi-gitter/internal/scm" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" ) type Config struct { From b5e70edac75058446a57e3dabd02260a51dda139 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Mon, 30 Oct 2023 12:16:21 -0700 Subject: [PATCH 12/24] feat: Modified repo-filter to repo-include and repo-exclude --- cmd/platform.go | 21 ++++++----- internal/scm/github/github.go | 56 ++++++++++++++++++++---------- internal/scm/github/github_test.go | 31 +++++++++++++++-- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index 95e782c6..a22f1199 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -28,7 +28,8 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.") flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") - flags.StringP("repo-filter", "", "", "Filter repositories with a Regular expression") + flags.StringP("repo-include", "", "", "Include repositories that match with a given Regular Expression") + flags.StringP("repo-exclude", "", "", "Exclude repositories that match with a given Regular Expression") flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") @@ -125,7 +126,8 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") repoSearch, _ := flag.GetString("repo-search") - repoFilter, _ := flag.GetString("repo-filter") + repoIncludeFilter, _ := flag.GetString("repo-include") + repoExcludeFilter, _ := flag.GetString("repo-exclude") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") @@ -167,13 +169,14 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu BaseURL: gitBaseURL, TransportMiddleware: http.NewLoggingRoundTripper, RepoListing: github.RepositoryListing{ - Organizations: orgs, - Users: users, - Repositories: repoRefs, - RepositorySearch: repoSearch, - Topics: topics, - SkipForks: skipForks, - RepositoryFilter: repoFilter, + Organizations: orgs, + Users: users, + Repositories: repoRefs, + RepositorySearch: repoSearch, + Topics: topics, + SkipForks: skipForks, + RepositoryIncludeFilter: repoIncludeFilter, + RepositoryExcludeFilter: repoExcludeFilter, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index be87eae6..6d1b80e1 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -7,7 +7,6 @@ import ( "net/http" "regexp" "slices" - "sort" "strings" "sync" "time" @@ -108,13 +107,16 @@ type Github struct { // RepositoryListing contains information about which repositories that should be fetched type RepositoryListing struct { - Organizations []string - Users []string - Repositories []RepositoryReference - RepositorySearch string - Topics []string - SkipForks bool - RepositoryFilter string + Organizations []string + Users []string + Repositories []RepositoryReference + RepositorySearch string + Topics []string + SkipForks bool + RepositoryIncludeFilter string + RepositoryExcludeFilter string + compiledRepositoryIncludeFilter *regexp.Regexp + compiledRepositoryExcludeFilter *regexp.Regexp } // RepositoryReference contains information to be able to reference a repository @@ -191,12 +193,28 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error) return repos, nil } +func (g *Github) excludeRepositoryFilter(repoName string) bool { + if g.RepositoryExcludeFilter == "" { + return false + } + if g.compiledRepositoryExcludeFilter == nil { + g.compiledRepositoryExcludeFilter = regexp.MustCompile(g.RepositoryExcludeFilter) + } + return g.compiledRepositoryExcludeFilter.MatchString(repoName) +} + +func (g *Github) matchesRepositoryFilter(repoName string) bool { + if g.RepositoryIncludeFilter == "" { + return true + } + if g.compiledRepositoryIncludeFilter == nil { + g.compiledRepositoryIncludeFilter = regexp.MustCompile(g.RepositoryIncludeFilter) + } + return g.compiledRepositoryIncludeFilter.MatchString(repoName) +} + func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) { allRepos := []*github.Repository{} - var repoRegEx *regexp.Regexp // repoFilter flag will filter all collected repositories - if g.RepositoryFilter != "" { - repoRegEx = regexp.MustCompile(g.RepositoryFilter) - } for _, org := range g.Organizations { repos, err := g.getOrganizationRepositories(ctx, org) if err != nil { @@ -228,23 +246,23 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repos...) } - filteredRepos := make([]*github.Repository, 0, len(allRepos)) // Filter repositories - filteredRepos = slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { - regExMatch := repoRegEx != nil && repoRegEx.MatchString(repo.GetFullName()) - return (!regExMatch && repoRegEx != nil) || (repo.GetArchived() || repo.GetDisabled()) + filteredRepos := slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { + regExMatch := g.matchesRepositoryFilter(repo.GetFullName()) + regExExclude := g.excludeRepositoryFilter(repo.GetFullName()) + return (!regExMatch || regExExclude) || (repo.GetArchived() || repo.GetDisabled()) }) // Remove duplicates - allRepos = make([]*github.Repository, 0, len(filteredRepos)) slices.SortFunc(filteredRepos, func(g2, g *github.Repository) int { return cmp.Compare(g2.GetFullName(), g.GetFullName()) }) + allRepos = slices.CompactFunc(filteredRepos, func(g2 *github.Repository, g *github.Repository) bool { return g2.GetFullName() == g.GetFullName() }) // Sort by Datetime - sort.Slice(allRepos, func(i, j int) bool { - return allRepos[i].GetCreatedAt().Before(allRepos[j].GetCreatedAt().Time) + slices.SortFunc(allRepos, func(g2, g *github.Repository) int { + return g2.GetCreatedAt().Time.Compare(g.GetCreatedAt().Time) }) return allRepos, nil } diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index ec133351..a2297a70 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -423,6 +423,32 @@ func Test_RepositoryFilter(t *testing.T) { "pull": true }, "created_at": "2020-01-03T16:49:19Z" + }, + { + "id": 3, + "name": "search-repo-3", + "full_name": "lindell/search-repo-3", + "private": false, + "topics": [ + "backend", + "go" + ], + "owner": { + "login": "lindell", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/lindell/search-repo-3", + "fork": true, + "archived": false, + "disabled": false, + "default_branch": "main", + "permissions": { + "admin": true, + "push": true, + "pull": true + }, + "created_at": "2020-01-03T16:49:19Z" } ] }`, @@ -432,8 +458,9 @@ func Test_RepositoryFilter(t *testing.T) { gh, err := github.New(github.Config{ TransportMiddleware: transport.Wrapper, RepoListing: github.RepositoryListing{ - RepositorySearch: "search-string", - RepositoryFilter: "search-repo(-)", + RepositorySearch: "search-string", + RepositoryIncludeFilter: "search-repo(-)", + RepositoryExcludeFilter: "search-repo-3$", }, MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, }) From 7d8d524a28944b245a719e8b859f4a622f87b8c8 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Mon, 6 Nov 2023 15:02:19 -0800 Subject: [PATCH 13/24] Using regex.Compile to raise regEx validation errors instead of panic --- cmd/platform.go | 8 ++-- internal/scm/github/github.go | 60 ++++++++++++++++++------------ internal/scm/github/github_test.go | 8 ++-- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index a22f1199..cb064027 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "regexp" "strings" "github.com/lindell/multi-gitter/internal/http" @@ -163,7 +164,8 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu if err != nil { return nil, err } - + repoIncludeFilterCompile, repoIncludeErr := regexp.Compile(repoIncludeFilter) + repoExcludeFilterCompile, repoExcludeErr := regexp.Compile(repoExcludeFilter) vc, err := github.New(github.Config{ Token: token, BaseURL: gitBaseURL, @@ -175,8 +177,8 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu RepositorySearch: repoSearch, Topics: topics, SkipForks: skipForks, - RepositoryIncludeFilter: repoIncludeFilter, - RepositoryExcludeFilter: repoExcludeFilter, + RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr}, + RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr}, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 6d1b80e1..0f690c3e 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -107,16 +107,14 @@ type Github struct { // RepositoryListing contains information about which repositories that should be fetched type RepositoryListing struct { - Organizations []string - Users []string - Repositories []RepositoryReference - RepositorySearch string - Topics []string - SkipForks bool - RepositoryIncludeFilter string - RepositoryExcludeFilter string - compiledRepositoryIncludeFilter *regexp.Regexp - compiledRepositoryExcludeFilter *regexp.Regexp + Organizations []string + Users []string + Repositories []RepositoryReference + RepositorySearch string + Topics []string + SkipForks bool + RepositoryIncludeFilter RepositoryFilter + RepositoryExcludeFilter RepositoryFilter } // RepositoryReference contains information to be able to reference a repository @@ -125,6 +123,12 @@ type RepositoryReference struct { Name string } +// RepositoryFilter contains repository filter information using regular expression +type RepositoryFilter struct { + Regex *regexp.Regexp + Err error +} + // String returns the string representation of a repo reference func (rr RepositoryReference) String() string { return fmt.Sprintf("%s/%s", rr.OwnerName, rr.Name) @@ -193,24 +197,32 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error) return repos, nil } -func (g *Github) excludeRepositoryFilter(repoName string) bool { - if g.RepositoryExcludeFilter == "" { - return false +func (g *Github) excludeRepositoryFilter(repoName string) (bool, error) { + if g.RepositoryExcludeFilter.Err != nil { + fmt.Println("repo-exclude RegEx Error: ", g.RepositoryExcludeFilter.Err) + return false, g.RepositoryExcludeFilter.Err } - if g.compiledRepositoryExcludeFilter == nil { - g.compiledRepositoryExcludeFilter = regexp.MustCompile(g.RepositoryExcludeFilter) + if g.RepositoryExcludeFilter.Regex == nil { + return false, nil } - return g.compiledRepositoryExcludeFilter.MatchString(repoName) + if g.RepositoryExcludeFilter.Regex.String() == "" { + return false, nil + } + return g.RepositoryExcludeFilter.Regex.MatchString(repoName), nil } -func (g *Github) matchesRepositoryFilter(repoName string) bool { - if g.RepositoryIncludeFilter == "" { - return true +func (g *Github) matchesRepositoryFilter(repoName string) (bool, error) { + if g.RepositoryIncludeFilter.Err != nil { + fmt.Println("repo-include RegEx Error: ", g.RepositoryIncludeFilter.Err) + return true, g.RepositoryIncludeFilter.Err + } + if g.RepositoryIncludeFilter.Regex == nil { + return true, nil } - if g.compiledRepositoryIncludeFilter == nil { - g.compiledRepositoryIncludeFilter = regexp.MustCompile(g.RepositoryIncludeFilter) + if g.RepositoryIncludeFilter.Regex.String() == "" { + return true, nil } - return g.compiledRepositoryIncludeFilter.MatchString(repoName) + return g.RepositoryIncludeFilter.Regex.MatchString(repoName), nil } func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) { @@ -248,8 +260,8 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } // Filter repositories filteredRepos := slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { - regExMatch := g.matchesRepositoryFilter(repo.GetFullName()) - regExExclude := g.excludeRepositoryFilter(repo.GetFullName()) + regExMatch, _ := g.matchesRepositoryFilter(repo.GetFullName()) + regExExclude, _ := g.excludeRepositoryFilter(repo.GetFullName()) return (!regExMatch || regExExclude) || (repo.GetArchived() || repo.GetDisabled()) }) // Remove duplicates diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index a2297a70..cfd660da 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/http" + "regexp" "strings" "testing" @@ -454,13 +455,14 @@ func Test_RepositoryFilter(t *testing.T) { }`, }, } - + repoIncludeFilterCompile, repoIncludeErr := regexp.Compile("search-repo(-)") + repoExcludeFilterCompile, repoExcludeErr := regexp.Compile("search-repo-3$") gh, err := github.New(github.Config{ TransportMiddleware: transport.Wrapper, RepoListing: github.RepositoryListing{ RepositorySearch: "search-string", - RepositoryIncludeFilter: "search-repo(-)", - RepositoryExcludeFilter: "search-repo-3$", + RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr}, + RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr}, }, MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, }) From c2f88348bcd366a66faa016a6905fd88dd1fa292 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Mon, 6 Nov 2023 15:25:39 -0800 Subject: [PATCH 14/24] Minor syntax changes from golangci --- internal/scm/github/github_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index cfd660da..2fbc082b 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -468,9 +468,8 @@ func Test_RepositoryFilter(t *testing.T) { }) require.NoError(t, err) - repos, err := gh.GetRepositories(context.Background()) + repos, _ := gh.GetRepositories(context.Background()) assert.Len(t, repos, 1) - } func Test_GetSearchRepository_TooManyResults(t *testing.T) { From 57fc0086476abd7f17645d7fedfd15f4f045bd6c Mon Sep 17 00:00:00 2001 From: josealdaco Date: Tue, 7 Nov 2023 13:12:32 -0800 Subject: [PATCH 15/24] Moving regex filtering away from github scm --- cmd/cmd-run.go | 58 ++++++++++------ cmd/platform.go | 19 ++--- internal/multigitter/run.go | 39 ++++++++--- internal/scm/github/github.go | 80 +++++---------------- internal/scm/github/github_test.go | 107 ----------------------------- 5 files changed, 91 insertions(+), 212 deletions(-) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index 031eb335..ce0e37d7 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/signal" + "regexp" "strings" "syscall" @@ -175,7 +176,22 @@ func run(cmd *cobra.Command, _ []string) error { <-c os.Exit(1) }() - + repoInclude, _ := flag.GetString("repo-include") + repoExclude, _ := flag.GetString("repo-exclude") + var repoIncludeFilterCompile *regexp.Regexp + var repoExcludeFilterCompile *regexp.Regexp + if repoInclude != "" { + repoIncludeFilterCompile, err = regexp.Compile(repoInclude) + if err != nil { + return err + } + } + if repoExclude != "" { + repoExcludeFilterCompile, err = regexp.Compile(repoExclude) + if err != nil { + return err + } + } runner := &multigitter.Runner{ ScriptPath: executablePath, Arguments: arguments, @@ -185,25 +201,27 @@ func run(cmd *cobra.Command, _ []string) error { VersionController: vc, - CommitMessage: commitMessage, - PullRequestTitle: prTitle, - PullRequestBody: prBody, - Reviewers: reviewers, - TeamReviewers: teamReviewers, - MaxReviewers: maxReviewers, - MaxTeamReviewers: maxTeamReviewers, - Interactive: interactive, - DryRun: dryRun, - Fork: forkMode, - ForkOwner: forkOwner, - SkipPullRequest: skipPullRequest, - SkipRepository: skipRepository, - CommitAuthor: commitAuthor, - BaseBranch: baseBranchName, - Assignees: assignees, - ConflictStrategy: conflictStrategy, - Draft: draft, - Labels: labels, + CommitMessage: commitMessage, + PullRequestTitle: prTitle, + PullRequestBody: prBody, + Reviewers: reviewers, + TeamReviewers: teamReviewers, + MaxReviewers: maxReviewers, + MaxTeamReviewers: maxTeamReviewers, + Interactive: interactive, + DryRun: dryRun, + Fork: forkMode, + ForkOwner: forkOwner, + SkipPullRequest: skipPullRequest, + SkipRepository: skipRepository, + RegExExcludeRepository: repoExcludeFilterCompile, + RegExIncludeRepository: repoIncludeFilterCompile, + CommitAuthor: commitAuthor, + BaseBranch: baseBranchName, + Assignees: assignees, + ConflictStrategy: conflictStrategy, + Draft: draft, + Labels: labels, Concurrent: concurrent, diff --git a/cmd/platform.go b/cmd/platform.go index cb064027..f233fdd2 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "regexp" "strings" "github.com/lindell/multi-gitter/internal/http" @@ -127,8 +126,6 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") repoSearch, _ := flag.GetString("repo-search") - repoIncludeFilter, _ := flag.GetString("repo-include") - repoExcludeFilter, _ := flag.GetString("repo-exclude") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") @@ -164,21 +161,17 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu if err != nil { return nil, err } - repoIncludeFilterCompile, repoIncludeErr := regexp.Compile(repoIncludeFilter) - repoExcludeFilterCompile, repoExcludeErr := regexp.Compile(repoExcludeFilter) vc, err := github.New(github.Config{ Token: token, BaseURL: gitBaseURL, TransportMiddleware: http.NewLoggingRoundTripper, RepoListing: github.RepositoryListing{ - Organizations: orgs, - Users: users, - Repositories: repoRefs, - RepositorySearch: repoSearch, - Topics: topics, - SkipForks: skipForks, - RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr}, - RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr}, + Organizations: orgs, + Users: users, + Repositories: repoRefs, + RepositorySearch: repoSearch, + Topics: topics, + SkipForks: skipForks, }, MergeTypes: mergeTypes, ForkMode: forkMode, diff --git a/internal/multigitter/run.go b/internal/multigitter/run.go index 5fc82106..6459ac4e 100755 --- a/internal/multigitter/run.go +++ b/internal/multigitter/run.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "os/exec" + "regexp" "sync" "syscall" @@ -54,12 +55,13 @@ type Runner struct { BaseBranch string // The base branch of the PR, use default branch if not set Assignees []string - Concurrent int - SkipPullRequest bool // If set, the script will run directly on the base-branch without creating any PR - SkipRepository []string // A list of repositories that run will skip - - Fork bool // If set, create a fork and make the pull request from it - ForkOwner string // The owner of the new fork. If empty, the fork should happen on the logged in user + Concurrent int + SkipPullRequest bool // If set, the script will run directly on the base-branch without creating any PR + SkipRepository []string // A list of repositories that run will skip + RegExIncludeRepository *regexp.Regexp + RegExExcludeRepository *regexp.Regexp + Fork bool // If set, create a fork and make the pull request from it + ForkOwner string // The owner of the new fork. If empty, the fork should happen on the logged in user ConflictStrategy ConflictStrategy // Defines what will happen if a branch already exists @@ -97,8 +99,7 @@ func (r *Runner) Run(ctx context.Context) error { if err != nil { return errors.Wrap(err, "could not fetch repositories") } - - repos = filterRepositories(repos, r.SkipRepository) + repos = filterRepositories(repos, r.SkipRepository, r.RegExIncludeRepository, r.RegExExcludeRepository) if len(repos) == 0 { log.Infof("No repositories found. Please make sure the user of the token has the correct access to the repos you want to change.") @@ -152,7 +153,24 @@ func (r *Runner) Run(ctx context.Context) error { return nil } -func filterRepositories(repos []scm.Repository, skipRepositoryNames []string) []scm.Repository { +// Determines if Repository should be excluded based on provided Regular Expression +func excludeRepositoryFilter(repoName string, regExp *regexp.Regexp) bool { + if regExp == nil { + return false + } + return regExp.MatchString(repoName) +} + +// Determines if Repository should be included based on provided Regular Expression +func matchesRepositoryFilter(repoName string, regExp *regexp.Regexp) bool { + if regExp == nil { + return true + } + return regExp.MatchString(repoName) +} + +func filterRepositories(repos []scm.Repository, skipRepositoryNames []string, regExIncludeRepository *regexp.Regexp, + regExExcludeRepository *regexp.Regexp) []scm.Repository { skipReposMap := map[string]struct{}{} for _, skipRepo := range skipRepositoryNames { skipReposMap[skipRepo] = struct{}{} @@ -160,7 +178,8 @@ func filterRepositories(repos []scm.Repository, skipRepositoryNames []string) [] filteredRepos := make([]scm.Repository, 0, len(repos)) for _, r := range repos { - if _, shouldSkip := skipReposMap[r.FullName()]; !shouldSkip { + if _, shouldSkip := skipReposMap[r.FullName()]; !shouldSkip && (matchesRepositoryFilter(r.FullName(), + regExIncludeRepository) && !excludeRepositoryFilter(r.FullName(), regExExcludeRepository)) { filteredRepos = append(filteredRepos, r) } else { log.Infof("Skipping %s", r.FullName()) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 51d5633d..083f6a09 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -1,12 +1,9 @@ package github import ( - "cmp" "context" "fmt" "net/http" - "regexp" - "slices" "strings" "sync" "time" @@ -107,14 +104,12 @@ type Github struct { // RepositoryListing contains information about which repositories that should be fetched type RepositoryListing struct { - Organizations []string - Users []string - Repositories []RepositoryReference - RepositorySearch string - Topics []string - SkipForks bool - RepositoryIncludeFilter RepositoryFilter - RepositoryExcludeFilter RepositoryFilter + Organizations []string + Users []string + Repositories []RepositoryReference + RepositorySearch string + Topics []string + SkipForks bool } // RepositoryReference contains information to be able to reference a repository @@ -123,12 +118,6 @@ type RepositoryReference struct { Name string } -// RepositoryFilter contains repository filter information using regular expression -type RepositoryFilter struct { - Regex *regexp.Regexp - Err error -} - // String returns the string representation of a repo reference func (rr RepositoryReference) String() string { return fmt.Sprintf("%s/%s", rr.OwnerName, rr.Name) @@ -197,34 +186,6 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error) return repos, nil } -func (g *Github) excludeRepositoryFilter(repoName string) (bool, error) { - if g.RepositoryExcludeFilter.Err != nil { - fmt.Println("repo-exclude RegEx Error: ", g.RepositoryExcludeFilter.Err) - return false, g.RepositoryExcludeFilter.Err - } - if g.RepositoryExcludeFilter.Regex == nil { - return false, nil - } - if g.RepositoryExcludeFilter.Regex.String() == "" { - return false, nil - } - return g.RepositoryExcludeFilter.Regex.MatchString(repoName), nil -} - -func (g *Github) matchesRepositoryFilter(repoName string) (bool, error) { - if g.RepositoryIncludeFilter.Err != nil { - fmt.Println("repo-include RegEx Error: ", g.RepositoryIncludeFilter.Err) - return true, g.RepositoryIncludeFilter.Err - } - if g.RepositoryIncludeFilter.Regex == nil { - return true, nil - } - if g.RepositoryIncludeFilter.Regex.String() == "" { - return true, nil - } - return g.RepositoryIncludeFilter.Regex.MatchString(repoName), nil -} - func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) { allRepos := []*github.Repository{} for _, org := range g.Organizations { @@ -258,24 +219,18 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repos...) } - // Filter repositories - filteredRepos := slices.DeleteFunc(allRepos, func(repo *github.Repository) bool { - regExMatch, _ := g.matchesRepositoryFilter(repo.GetFullName()) - regExExclude, _ := g.excludeRepositoryFilter(repo.GetFullName()) - return (!regExMatch || regExExclude) || (repo.GetArchived() || repo.GetDisabled()) - }) // Remove duplicates - slices.SortFunc(filteredRepos, func(g2, g *github.Repository) int { - return cmp.Compare(g2.GetFullName(), g.GetFullName()) - }) - - allRepos = slices.CompactFunc(filteredRepos, func(g2 *github.Repository, g *github.Repository) bool { - return g2.GetFullName() == g.GetFullName() - }) - // Sort by Datetime - slices.SortFunc(allRepos, func(g2, g *github.Repository) int { - return g2.GetCreatedAt().Time.Compare(g.GetCreatedAt().Time) - }) + repoMap := map[string]*github.Repository{} + for _, repo := range allRepos { + repoMap[repo.GetFullName()] = repo + } + allRepos = make([]*github.Repository, 0, len(repoMap)) + for _, repo := range repoMap { + if repo.GetArchived() || repo.GetDisabled() { + continue + } + allRepos = append(allRepos, repo) + } return allRepos, nil } @@ -344,6 +299,7 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g if err != nil { return nil, nil, err } + if rr.GetIncompleteResults() { // can occur when search times out on the server: for now, fail instead // of handling the issue diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index 2fbc082b..0be69fc4 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -5,7 +5,6 @@ import ( "errors" "io" "net/http" - "regexp" "strings" "testing" @@ -366,112 +365,6 @@ func Test_GetSearchRepository_Incomplete(t *testing.T) { assert.Len(t, repos, 0) } -func Test_RepositoryFilter(t *testing.T) { - transport := testTransport{ - pathBodies: map[string]string{ - "/search/repositories": `{ - "total_count": 2, - "incomplete_results": false, - "items": [ - { - "id": 1, - "name": "search-repo1", - "full_name": "lindell/search-repo1", - "private": false, - "topics": [ - "backend", - "go" - ], - "owner": { - "login": "lindell", - "type": "User", - "site_admin": false - }, - "html_url": "https://github.com/lindell/search-repo1", - "fork": true, - "archived": false, - "disabled": false, - "default_branch": "main", - "permissions": { - "admin": true, - "push": true, - "pull": true - }, - "created_at": "2020-01-03T16:49:19Z" - }, - { - "id": 2, - "name": "search-repo-2", - "full_name": "lindell/search-repo-2", - "private": false, - "topics": [ - "backend", - "go" - ], - "owner": { - "login": "lindell", - "type": "User", - "site_admin": false - }, - "html_url": "https://github.com/lindell/search-repo-2", - "fork": true, - "archived": false, - "disabled": false, - "default_branch": "main", - "permissions": { - "admin": true, - "push": true, - "pull": true - }, - "created_at": "2020-01-03T16:49:19Z" - }, - { - "id": 3, - "name": "search-repo-3", - "full_name": "lindell/search-repo-3", - "private": false, - "topics": [ - "backend", - "go" - ], - "owner": { - "login": "lindell", - "type": "User", - "site_admin": false - }, - "html_url": "https://github.com/lindell/search-repo-3", - "fork": true, - "archived": false, - "disabled": false, - "default_branch": "main", - "permissions": { - "admin": true, - "push": true, - "pull": true - }, - "created_at": "2020-01-03T16:49:19Z" - } - ] - }`, - }, - } - repoIncludeFilterCompile, repoIncludeErr := regexp.Compile("search-repo(-)") - repoExcludeFilterCompile, repoExcludeErr := regexp.Compile("search-repo-3$") - gh, err := github.New(github.Config{ - TransportMiddleware: transport.Wrapper, - RepoListing: github.RepositoryListing{ - RepositorySearch: "search-string", - RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr}, - RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr}, - }, - MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, - }) - require.NoError(t, err) - - repos, _ := gh.GetRepositories(context.Background()) - assert.Len(t, repos, 1) -} - func Test_GetSearchRepository_TooManyResults(t *testing.T) { transport := testTransport{ pathBodies: map[string]string{ From d1082cbee187e17a0d51999817a790c17b9be6e7 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Tue, 7 Nov 2023 13:16:27 -0800 Subject: [PATCH 16/24] reverting sort duplication change --- internal/scm/github/github.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 083f6a09..38b5cabf 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sort" "strings" "sync" "time" @@ -219,7 +220,7 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repos...) } - // Remove duplicates + // Remove duplicate repos repoMap := map[string]*github.Repository{} for _, repo := range allRepos { repoMap[repo.GetFullName()] = repo @@ -231,6 +232,10 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repo) } + sort.Slice(allRepos, func(i, j int) bool { + return allRepos[i].GetCreatedAt().Before(allRepos[j].GetCreatedAt().Time) + }) + return allRepos, nil } @@ -322,6 +327,7 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g } i++ } + return repos, nil } From a452b403b885d7d6e2262c0bf77682b82e13b887 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Tue, 7 Nov 2023 13:17:34 -0800 Subject: [PATCH 17/24] removing changes to .github.go file --- internal/scm/github/github.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 38b5cabf..10a1c6f7 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -189,6 +189,7 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error) func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) { allRepos := []*github.Repository{} + for _, org := range g.Organizations { repos, err := g.getOrganizationRepositories(ctx, org) if err != nil { @@ -220,6 +221,7 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err } allRepos = append(allRepos, repos...) } + // Remove duplicate repos repoMap := map[string]*github.Repository{} for _, repo := range allRepos { From be71b2e51275840b382d0b86ec5b3ef4385f8adf Mon Sep 17 00:00:00 2001 From: josealdaco Date: Tue, 7 Nov 2023 15:30:12 -0800 Subject: [PATCH 18/24] Including unit test for feature --- tests/table_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/table_test.go b/tests/table_test.go index a6199e36..65299ee0 100644 --- a/tests/table_test.go +++ b/tests/table_test.go @@ -336,6 +336,32 @@ func TestTable(t *testing.T) { assert.False(t, branchExist(t, vcMock.Repositories[0].Path, "custom-branch-name")) }, }, + { + name: "regex repository filtering", + vcCreate: func(t *testing.T) *vcmock.VersionController { + return &vcmock.VersionController{ + Repositories: []vcmock.Repository{ + createRepo(t, "owner", "repo1", "i like apples"), + createRepo(t, "owner", "repo-2", "i like oranges"), + createRepo(t, "owner", "repo-change", "i like carrots"), + createRepo(t, "owner", "repo-3", "i like carrots"), + }, + } + }, + args: []string{ + "run", + "--repo-search", "repo", + "--repo-include", "^owner/repo-", + "--repo-exclude", "\\d$", + "--commit-message", "chore: foo", + "--dry-run", + changerBinaryPath, + }, + verify: func(t *testing.T, vcMock *vcmock.VersionController, runData runData) { + require.Len(t, vcMock.PullRequests, 0) + assert.Contains(t, runData.logOut, "Running on 1 repositories") + }, + }, { name: "parallel", From d368cc28a8e261d3d20d4916d665fc916b70df12 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Wed, 8 Nov 2023 09:07:03 -0800 Subject: [PATCH 19/24] including more tests for regex & wrapping errors --- cmd/cmd-run.go | 6 +-- internal/multigitter/run.go | 11 ++++-- tests/table_test.go | 78 ++++++++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index ce0e37d7..ee2eed39 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "errors" "fmt" "os" "os/signal" @@ -13,6 +12,7 @@ import ( "github.com/lindell/multi-gitter/internal/git" "github.com/lindell/multi-gitter/internal/multigitter" + "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -183,13 +183,13 @@ func run(cmd *cobra.Command, _ []string) error { if repoInclude != "" { repoIncludeFilterCompile, err = regexp.Compile(repoInclude) if err != nil { - return err + return errors.WithMessage(err, "could not parse repo-include") } } if repoExclude != "" { repoExcludeFilterCompile, err = regexp.Compile(repoExclude) if err != nil { - return err + return errors.WithMessage(err, "could not parse repo-exclude") } } runner := &multigitter.Runner{ diff --git a/internal/multigitter/run.go b/internal/multigitter/run.go index 6459ac4e..29f57055 100755 --- a/internal/multigitter/run.go +++ b/internal/multigitter/run.go @@ -178,11 +178,14 @@ func filterRepositories(repos []scm.Repository, skipRepositoryNames []string, re filteredRepos := make([]scm.Repository, 0, len(repos)) for _, r := range repos { - if _, shouldSkip := skipReposMap[r.FullName()]; !shouldSkip && (matchesRepositoryFilter(r.FullName(), - regExIncludeRepository) && !excludeRepositoryFilter(r.FullName(), regExExcludeRepository)) { - filteredRepos = append(filteredRepos, r) + if _, shouldSkip := skipReposMap[r.FullName()]; shouldSkip { + log.Infof("Skipping %s since it is in exclusion list", r.FullName()) + } else if !matchesRepositoryFilter(r.FullName(), regExIncludeRepository) { + log.Infof("Skipping %s since it does not match the inclusion regexp", r.FullName()) + } else if excludeRepositoryFilter(r.FullName(), regExExcludeRepository) { + log.Infof("Skipping %s since it match the exclusion regexp", r.FullName()) } else { - log.Infof("Skipping %s", r.FullName()) + filteredRepos = append(filteredRepos, r) } } return filteredRepos diff --git a/tests/table_test.go b/tests/table_test.go index 65299ee0..d6462911 100644 --- a/tests/table_test.go +++ b/tests/table_test.go @@ -337,7 +337,7 @@ func TestTable(t *testing.T) { }, }, { - name: "regex repository filtering", + name: "repo-include regex repository filtering", vcCreate: func(t *testing.T) *vcmock.VersionController { return &vcmock.VersionController{ Repositories: []vcmock.Repository{ @@ -352,6 +352,30 @@ func TestTable(t *testing.T) { "run", "--repo-search", "repo", "--repo-include", "^owner/repo-", + "--commit-message", "chore: foo", + "--dry-run", + changerBinaryPath, + }, + verify: func(t *testing.T, vcMock *vcmock.VersionController, runData runData) { + require.Len(t, vcMock.PullRequests, 0) + assert.Contains(t, runData.logOut, "Running on 3 repositories") + }, + }, + { + name: "repo-exclude regex repository filtering", + vcCreate: func(t *testing.T) *vcmock.VersionController { + return &vcmock.VersionController{ + Repositories: []vcmock.Repository{ + createRepo(t, "owner", "repo1", "i like apples"), + createRepo(t, "owner", "repo-2", "i like oranges"), + createRepo(t, "owner", "repo-change", "i like carrots"), + createRepo(t, "owner", "repo-3", "i like carrots"), + }, + } + }, + args: []string{ + "run", + "--repo-search", "repo", "--repo-exclude", "\\d$", "--commit-message", "chore: foo", "--dry-run", @@ -362,6 +386,58 @@ func TestTable(t *testing.T) { assert.Contains(t, runData.logOut, "Running on 1 repositories") }, }, + { + name: "invalid repo-include regex repository filtering", + vcCreate: func(t *testing.T) *vcmock.VersionController { + return &vcmock.VersionController{ + Repositories: []vcmock.Repository{ + createRepo(t, "owner", "repo1", "i like apples"), + createRepo(t, "owner", "repo-2", "i like oranges"), + createRepo(t, "owner", "repo-change", "i like carrots"), + createRepo(t, "owner", "repo-3", "i like carrots"), + }, + } + }, + args: []string{ + "run", + "--repo-search", "repo", + "--repo-include", "(abc[def$", + "--commit-message", "chore: foo", + "--dry-run", + changerBinaryPath, + }, + verify: func(t *testing.T, vcMock *vcmock.VersionController, runData runData) { + require.Len(t, vcMock.PullRequests, 0) + assert.Contains(t, runData.cmdOut, "could not parse repo-include") + }, + expectErr: true, + }, + { + name: "invalid repo-exclude regex repository filtering", + vcCreate: func(t *testing.T) *vcmock.VersionController { + return &vcmock.VersionController{ + Repositories: []vcmock.Repository{ + createRepo(t, "owner", "repo1", "i like apples"), + createRepo(t, "owner", "repo-2", "i like oranges"), + createRepo(t, "owner", "repo-change", "i like carrots"), + createRepo(t, "owner", "repo-3", "i like carrots"), + }, + } + }, + args: []string{ + "run", + "--repo-search", "repo", + "--repo-exclude", "(abc[def$", + "--commit-message", "chore: foo", + "--dry-run", + changerBinaryPath, + }, + verify: func(t *testing.T, vcMock *vcmock.VersionController, runData runData) { + require.Len(t, vcMock.PullRequests, 0) + assert.Contains(t, runData.cmdOut, "could not parse repo-exclude") + }, + expectErr: true, + }, { name: "parallel", From 696d38863553912e9b23f0814980089ff500e84b Mon Sep 17 00:00:00 2001 From: josealdaco Date: Thu, 9 Nov 2023 10:08:48 -0800 Subject: [PATCH 20/24] chore: minor style fixes --- cmd/cmd-run.go | 75 ++++++++++++++++++------------------- cmd/platform.go | 2 - internal/multigitter/run.go | 5 ++- 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index ee2eed39..8fdfe740 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -63,6 +63,8 @@ Available values: cmd.Flags().StringSliceP("labels", "", nil, "Labels to be added to any created pull request.") cmd.Flags().StringP("author-name", "", "", "Name of the committer. If not set, the global git config setting will be used.") cmd.Flags().StringP("author-email", "", "", "Email of the committer. If not set, the global git config setting will be used.") + cmd.Flags().StringP("repo-include", "", "", "Include repositories that match with a given Regular Expression") + cmd.Flags().StringP("repo-exclude", "", "", "Exclude repositories that match with a given Regular Expression") configureGit(cmd) configurePlatform(cmd) configureRunPlatform(cmd, true) @@ -99,6 +101,8 @@ func run(cmd *cobra.Command, _ []string) error { assignees, _ := flag.GetStringSlice("assignees") draft, _ := flag.GetBool("draft") labels, _ := flag.GetStringSlice("labels") + repoInclude, _ := flag.GetString("repo-include") + repoExclude, _ := flag.GetString("repo-exclude") if concurrent < 1 { return errors.New("concurrent runs can't be less than one") @@ -176,22 +180,6 @@ func run(cmd *cobra.Command, _ []string) error { <-c os.Exit(1) }() - repoInclude, _ := flag.GetString("repo-include") - repoExclude, _ := flag.GetString("repo-exclude") - var repoIncludeFilterCompile *regexp.Regexp - var repoExcludeFilterCompile *regexp.Regexp - if repoInclude != "" { - repoIncludeFilterCompile, err = regexp.Compile(repoInclude) - if err != nil { - return errors.WithMessage(err, "could not parse repo-include") - } - } - if repoExclude != "" { - repoExcludeFilterCompile, err = regexp.Compile(repoExclude) - if err != nil { - return errors.WithMessage(err, "could not parse repo-exclude") - } - } runner := &multigitter.Runner{ ScriptPath: executablePath, Arguments: arguments, @@ -201,33 +189,44 @@ func run(cmd *cobra.Command, _ []string) error { VersionController: vc, - CommitMessage: commitMessage, - PullRequestTitle: prTitle, - PullRequestBody: prBody, - Reviewers: reviewers, - TeamReviewers: teamReviewers, - MaxReviewers: maxReviewers, - MaxTeamReviewers: maxTeamReviewers, - Interactive: interactive, - DryRun: dryRun, - Fork: forkMode, - ForkOwner: forkOwner, - SkipPullRequest: skipPullRequest, - SkipRepository: skipRepository, - RegExExcludeRepository: repoExcludeFilterCompile, - RegExIncludeRepository: repoIncludeFilterCompile, - CommitAuthor: commitAuthor, - BaseBranch: baseBranchName, - Assignees: assignees, - ConflictStrategy: conflictStrategy, - Draft: draft, - Labels: labels, + CommitMessage: commitMessage, + PullRequestTitle: prTitle, + PullRequestBody: prBody, + Reviewers: reviewers, + TeamReviewers: teamReviewers, + MaxReviewers: maxReviewers, + MaxTeamReviewers: maxTeamReviewers, + Interactive: interactive, + DryRun: dryRun, + Fork: forkMode, + ForkOwner: forkOwner, + SkipPullRequest: skipPullRequest, + SkipRepository: skipRepository, + CommitAuthor: commitAuthor, + BaseBranch: baseBranchName, + Assignees: assignees, + ConflictStrategy: conflictStrategy, + Draft: draft, + Labels: labels, Concurrent: concurrent, CreateGit: gitCreator, } - + if repoInclude != "" { + repoIncludeFilterCompile, err := regexp.Compile(repoInclude) + if err != nil { + return errors.WithMessage(err, "could not parse repo-include") + } + runner.RegExIncludeRepository = repoIncludeFilterCompile + } + if repoExclude != "" { + repoExcludeFilterCompile, err := regexp.Compile(repoExclude) + if err != nil { + return errors.WithMessage(err, "could not parse repo-exclude") + } + runner.RegExExcludeRepository = repoExcludeFilterCompile + } err = runner.Run(ctx) if err != nil { fmt.Println(err.Error()) diff --git a/cmd/platform.go b/cmd/platform.go index f233fdd2..592ea668 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -28,8 +28,6 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.") flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") - flags.StringP("repo-include", "", "", "Include repositories that match with a given Regular Expression") - flags.StringP("repo-exclude", "", "", "Exclude repositories that match with a given Regular Expression") flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") diff --git a/internal/multigitter/run.go b/internal/multigitter/run.go index 29f57055..33138ed5 100755 --- a/internal/multigitter/run.go +++ b/internal/multigitter/run.go @@ -60,8 +60,9 @@ type Runner struct { SkipRepository []string // A list of repositories that run will skip RegExIncludeRepository *regexp.Regexp RegExExcludeRepository *regexp.Regexp - Fork bool // If set, create a fork and make the pull request from it - ForkOwner string // The owner of the new fork. If empty, the fork should happen on the logged in user + + Fork bool // If set, create a fork and make the pull request from it + ForkOwner string // The owner of the new fork. If empty, the fork should happen on the logged in user ConflictStrategy ConflictStrategy // Defines what will happen if a branch already exists From b41d462b1362e48a6438b36818dc9f8a322cf81f Mon Sep 17 00:00:00 2001 From: josealdaco Date: Thu, 9 Nov 2023 10:13:03 -0800 Subject: [PATCH 21/24] chore: remove changes to platform file --- cmd/platform.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/platform.go b/cmd/platform.go index 592ea668..e63824d1 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -129,6 +129,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu forkOwner, _ := flag.GetString("fork-owner") sshAuth, _ := flag.GetBool("ssh-auth") skipForks, _ := flag.GetBool("skip-forks") + if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" { return nil, errors.New("no organization, user, repo or repo-search set") } @@ -159,6 +160,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu if err != nil { return nil, err } + vc, err := github.New(github.Config{ Token: token, BaseURL: gitBaseURL, From 06c8ad8b914ea3bb2e7a7b805da0b50a418ad81c Mon Sep 17 00:00:00 2001 From: josealdaco Date: Thu, 9 Nov 2023 12:49:50 -0800 Subject: [PATCH 22/24] fix:moving regex compiling --- cmd/cmd-run.go | 73 +++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index 8fdfe740..cbf3d6df 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -148,7 +148,22 @@ func run(cmd *cobra.Command, _ []string) error { Email: authorEmail, } } - + var regExIncludeRepository *regexp.Regexp + var regExExcludeRepository *regexp.Regexp + if repoInclude != "" { + repoIncludeFilterCompile, err := regexp.Compile(repoInclude) + if err != nil { + return errors.WithMessage(err, "could not parse repo-include") + } + regExIncludeRepository = repoIncludeFilterCompile + } + if repoExclude != "" { + repoExcludeFilterCompile, err := regexp.Compile(repoExclude) + if err != nil { + return errors.WithMessage(err, "could not parse repo-exclude") + } + regExExcludeRepository = repoExcludeFilterCompile + } vc, err := getVersionController(flag, true, false) if err != nil { return err @@ -180,6 +195,7 @@ func run(cmd *cobra.Command, _ []string) error { <-c os.Exit(1) }() + runner := &multigitter.Runner{ ScriptPath: executablePath, Arguments: arguments, @@ -189,44 +205,33 @@ func run(cmd *cobra.Command, _ []string) error { VersionController: vc, - CommitMessage: commitMessage, - PullRequestTitle: prTitle, - PullRequestBody: prBody, - Reviewers: reviewers, - TeamReviewers: teamReviewers, - MaxReviewers: maxReviewers, - MaxTeamReviewers: maxTeamReviewers, - Interactive: interactive, - DryRun: dryRun, - Fork: forkMode, - ForkOwner: forkOwner, - SkipPullRequest: skipPullRequest, - SkipRepository: skipRepository, - CommitAuthor: commitAuthor, - BaseBranch: baseBranchName, - Assignees: assignees, - ConflictStrategy: conflictStrategy, - Draft: draft, - Labels: labels, + CommitMessage: commitMessage, + PullRequestTitle: prTitle, + PullRequestBody: prBody, + Reviewers: reviewers, + TeamReviewers: teamReviewers, + MaxReviewers: maxReviewers, + MaxTeamReviewers: maxTeamReviewers, + Interactive: interactive, + DryRun: dryRun, + RegExIncludeRepository: regExIncludeRepository, + RegExExcludeRepository: regExExcludeRepository, + Fork: forkMode, + ForkOwner: forkOwner, + SkipPullRequest: skipPullRequest, + SkipRepository: skipRepository, + CommitAuthor: commitAuthor, + BaseBranch: baseBranchName, + Assignees: assignees, + ConflictStrategy: conflictStrategy, + Draft: draft, + Labels: labels, Concurrent: concurrent, CreateGit: gitCreator, } - if repoInclude != "" { - repoIncludeFilterCompile, err := regexp.Compile(repoInclude) - if err != nil { - return errors.WithMessage(err, "could not parse repo-include") - } - runner.RegExIncludeRepository = repoIncludeFilterCompile - } - if repoExclude != "" { - repoExcludeFilterCompile, err := regexp.Compile(repoExclude) - if err != nil { - return errors.WithMessage(err, "could not parse repo-exclude") - } - runner.RegExExcludeRepository = repoExcludeFilterCompile - } + err = runner.Run(ctx) if err != nil { fmt.Println(err.Error()) From b4ff2b2b78c25e4c09fe048d3a41b6799857dee1 Mon Sep 17 00:00:00 2001 From: josealdaco Date: Thu, 9 Nov 2023 12:53:58 -0800 Subject: [PATCH 23/24] fix:undo removing unrelated linebreaks --- cmd/cmd-run.go | 1 + internal/multigitter/run.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index cbf3d6df..e69862be 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -148,6 +148,7 @@ func run(cmd *cobra.Command, _ []string) error { Email: authorEmail, } } + var regExIncludeRepository *regexp.Regexp var regExExcludeRepository *regexp.Regexp if repoInclude != "" { diff --git a/internal/multigitter/run.go b/internal/multigitter/run.go index 33138ed5..1d84dfb3 100755 --- a/internal/multigitter/run.go +++ b/internal/multigitter/run.go @@ -100,6 +100,7 @@ func (r *Runner) Run(ctx context.Context) error { if err != nil { return errors.Wrap(err, "could not fetch repositories") } + repos = filterRepositories(repos, r.SkipRepository, r.RegExIncludeRepository, r.RegExExcludeRepository) if len(repos) == 0 { From 9ec419122716978895d9a82860061cd7c2156938 Mon Sep 17 00:00:00 2001 From: Johan Lindell Date: Thu, 9 Nov 2023 21:56:49 +0100 Subject: [PATCH 24/24] Added newline between sections --- cmd/cmd-run.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/cmd-run.go b/cmd/cmd-run.go index e69862be..d4ed2478 100755 --- a/cmd/cmd-run.go +++ b/cmd/cmd-run.go @@ -165,6 +165,7 @@ func run(cmd *cobra.Command, _ []string) error { } regExExcludeRepository = repoExcludeFilterCompile } + vc, err := getVersionController(flag, true, false) if err != nil { return err