feat: Including Regular Expression filter when targeting Github Repositories

cmd/platform.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/lindell/multi-gitter/internal/http"
@@ -28,6 +29,8 @@ func configurePlatform(cmd *cobra.Command) {
 	flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.")
 	flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.")
 	flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".")
+	flags.StringP("repo-include", "", "", "Include repositories that match with a given Regular Expression")
+	flags.StringP("repo-exclude", "", "", "Exclude repositories that match with a given Regular Expression")
 	flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories")
 	flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.")
 	flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".")
@@ -124,12 +127,13 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu
 	users, _ := flag.GetStringSlice("user")
 	repos, _ := flag.GetStringSlice("repo")
 	repoSearch, _ := flag.GetString("repo-search")
+	repoIncludeFilter, _ := flag.GetString("repo-include")
+	repoExcludeFilter, _ := flag.GetString("repo-exclude")
 	topics, _ := flag.GetStringSlice("topic")
 	forkMode, _ := flag.GetBool("fork")
 	forkOwner, _ := flag.GetString("fork-owner")
 	sshAuth, _ := flag.GetBool("ssh-auth")
 	skipForks, _ := flag.GetBool("skip-forks")
-
 	if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" {
 		return nil, errors.New("no organization, user, repo or repo-search set")
 	}
@@ -160,18 +164,21 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu
 	if err != nil {
 		return nil, err
 	}
-
+	repoIncludeFilterCompile, repoIncludeErr := regexp.Compile(repoIncludeFilter)
+	repoExcludeFilterCompile, repoExcludeErr := regexp.Compile(repoExcludeFilter)
 	vc, err := github.New(github.Config{
 		Token:               token,
 		BaseURL:             gitBaseURL,
 		TransportMiddleware: http.NewLoggingRoundTripper,
 		RepoListing: github.RepositoryListing{
-			Organizations:    orgs,
-			Users:            users,
-			Repositories:     repoRefs,
-			RepositorySearch: repoSearch,
-			Topics:           topics,
-			SkipForks:        skipForks,
+			Organizations:           orgs,
+			Users:                   users,
+			Repositories:            repoRefs,
+			RepositorySearch:        repoSearch,
+			Topics:                  topics,
+			SkipForks:               skipForks,
+			RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr},
+			RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr},
 		},
 		MergeTypes:       mergeTypes,
 		ForkMode:         forkMode,

internal/scm/github/github.go

@@ -1,10 +1,12 @@
 package github
 
 import (
+	"cmp"
 	"context"
 	"fmt"
 	"net/http"
-	"sort"
+	"regexp"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -105,12 +107,14 @@ type Github struct {
 
 // RepositoryListing contains information about which repositories that should be fetched
 type RepositoryListing struct {
-	Organizations    []string
-	Users            []string
-	Repositories     []RepositoryReference
-	RepositorySearch string
-	Topics           []string
-	SkipForks        bool
+	Organizations           []string
+	Users                   []string
+	Repositories            []RepositoryReference
+	RepositorySearch        string
+	Topics                  []string
+	SkipForks               bool
+	RepositoryIncludeFilter RepositoryFilter
+	RepositoryExcludeFilter RepositoryFilter
 }
 
 // RepositoryReference contains information to be able to reference a repository
@@ -119,6 +123,12 @@ type RepositoryReference struct {
 	Name      string
 }
 
+// RepositoryFilter contains repository filter information using regular expression
+type RepositoryFilter struct {
+	Regex *regexp.Regexp
+	Err   error
+}
+
 // String returns the string representation of a repo reference
 func (rr RepositoryReference) String() string {
 	return fmt.Sprintf("%s/%s", rr.OwnerName, rr.Name)
@@ -187,9 +197,36 @@ func (g *Github) GetRepositories(ctx context.Context) ([]scm.Repository, error)
 	return repos, nil
 }
 
+func (g *Github) excludeRepositoryFilter(repoName string) (bool, error) {
+	if g.RepositoryExcludeFilter.Err != nil {
+		fmt.Println("repo-exclude RegEx Error: ", g.RepositoryExcludeFilter.Err)
+		return false, g.RepositoryExcludeFilter.Err
+	}
+	if g.RepositoryExcludeFilter.Regex == nil {
+		return false, nil
+	}
+	if g.RepositoryExcludeFilter.Regex.String() == "" {
+		return false, nil
+	}
+	return g.RepositoryExcludeFilter.Regex.MatchString(repoName), nil
+}
+
+func (g *Github) matchesRepositoryFilter(repoName string) (bool, error) {
+	if g.RepositoryIncludeFilter.Err != nil {
+		fmt.Println("repo-include RegEx Error: ", g.RepositoryIncludeFilter.Err)
+		return true, g.RepositoryIncludeFilter.Err
+	}
+	if g.RepositoryIncludeFilter.Regex == nil {
+		return true, nil
+	}
+	if g.RepositoryIncludeFilter.Regex.String() == "" {
+		return true, nil
+	}
+	return g.RepositoryIncludeFilter.Regex.MatchString(repoName), nil
+}
+
 func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, error) {
 	allRepos := []*github.Repository{}
-
 	for _, org := range g.Organizations {
 		repos, err := g.getOrganizationRepositories(ctx, org)
 		if err != nil {
@@ -221,23 +258,24 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err
 		}
 		allRepos = append(allRepos, repos...)
 	}
-
-	// Remove duplicate repos
-	repoMap := map[string]*github.Repository{}
-	for _, repo := range allRepos {
-		repoMap[repo.GetFullName()] = repo
-	}
-	allRepos = make([]*github.Repository, 0, len(repoMap))
-	for _, repo := range repoMap {
-		if repo.GetArchived() || repo.GetDisabled() {
-			continue
-		}
-		allRepos = append(allRepos, repo)
-	}
-	sort.Slice(allRepos, func(i, j int) bool {
-		return allRepos[i].GetCreatedAt().Before(allRepos[j].GetCreatedAt().Time)
+	// Filter repositories
+	filteredRepos := slices.DeleteFunc(allRepos, func(repo *github.Repository) bool {
+		regExMatch, _ := g.matchesRepositoryFilter(repo.GetFullName())
+		regExExclude, _ := g.excludeRepositoryFilter(repo.GetFullName())
+		return (!regExMatch || regExExclude) || (repo.GetArchived() || repo.GetDisabled())
+	})
+	// Remove duplicates
+	slices.SortFunc(filteredRepos, func(g2, g *github.Repository) int {
+		return cmp.Compare(g2.GetFullName(), g.GetFullName())
 	})
 
+	allRepos = slices.CompactFunc(filteredRepos, func(g2 *github.Repository, g *github.Repository) bool {
+		return g2.GetFullName() == g.GetFullName()
+	})
+	// Sort by Datetime
+	slices.SortFunc(allRepos, func(g2, g *github.Repository) int {
+		return g2.GetCreatedAt().Time.Compare(g.GetCreatedAt().Time)
+	})
 	return allRepos, nil
 }
 
@@ -306,7 +344,6 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g
 			if err != nil {
 				return nil, nil, err
 			}
-
 			if rr.GetIncompleteResults() {
 				// can occur when search times out on the server: for now, fail instead
 				// of handling the issue
@@ -329,7 +366,6 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g
 		}
 		i++
 	}
-
 	return repos, nil
 }
 

internal/scm/github/github_test.go

@@ -5,6 +5,7 @@
 	"errors"
 	"io"
 	"net/http"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -365,6 +366,112 @@
 	assert.Len(t, repos, 0)
 }
 
+func Test_RepositoryFilter(t *testing.T) {
+	transport := testTransport{
+		pathBodies: map[string]string{
+			"/search/repositories": `{
+					"total_count": 2,
+					"incomplete_results": false,
+					"items": [
+					{
+						"id": 1,
+						"name": "search-repo1",
+						"full_name": "lindell/search-repo1",
+						"private": false,
+						"topics": [
+							"backend",
+							"go"
+						],
+						"owner": {
+							"login": "lindell",
+							"type": "User",
+							"site_admin": false
+						},
+						"html_url": "https://github.com/lindell/search-repo1",
+						"fork": true,
+						"archived": false,
+						"disabled": false,
+						"default_branch": "main",
+						"permissions": {
+							"admin": true,
+							"push": true,
+							"pull": true
+						},
+						"created_at": "2020-01-03T16:49:19Z"
+					},
+					{
+						"id": 2,
+						"name": "search-repo-2",
+						"full_name": "lindell/search-repo-2",
+						"private": false,
+						"topics": [
+							"backend",
+							"go"
+						],
+						"owner": {
+							"login": "lindell",
+							"type": "User",
+							"site_admin": false
+						},
+						"html_url": "https://github.com/lindell/search-repo-2",
+						"fork": true,
+						"archived": false,
+						"disabled": false,
+						"default_branch": "main",
+						"permissions": {
+							"admin": true,
+							"push": true,
+							"pull": true
+						},
+						"created_at": "2020-01-03T16:49:19Z"
+					},
+					{
+						"id": 3,
+						"name": "search-repo-3",
+						"full_name": "lindell/search-repo-3",
+						"private": false,
+						"topics": [
+							"backend",
+							"go"
+						],
+						"owner": {
+							"login": "lindell",
+							"type": "User",
+							"site_admin": false
+						},
+						"html_url": "https://github.com/lindell/search-repo-3",
+						"fork": true,
+						"archived": false,
+						"disabled": false,
+						"default_branch": "main",
+						"permissions": {
+							"admin": true,
+							"push": true,
+							"pull": true
+						},
+						"created_at": "2020-01-03T16:49:19Z"
+					}
+				]
+			}`,
+		},
+	}
+	repoIncludeFilterCompile, repoIncludeErr := regexp.Compile("search-repo(-)")
+	repoExcludeFilterCompile, repoExcludeErr := regexp.Compile("search-repo-3$")
+	gh, err := github.New(github.Config{
+		TransportMiddleware: transport.Wrapper,
+		RepoListing: github.RepositoryListing{
+			RepositorySearch:        "search-string",
+			RepositoryIncludeFilter: github.RepositoryFilter{Regex: repoIncludeFilterCompile, Err: repoIncludeErr},
+			RepositoryExcludeFilter: github.RepositoryFilter{Regex: repoExcludeFilterCompile, Err: repoExcludeErr},
+		},
+		MergeTypes: []scm.MergeType{scm.MergeTypeMerge},
+	})
+	require.NoError(t, err)
+
+	repos, _ := gh.GetRepositories(context.Background())
+	assert.Len(t, repos, 1)
+}
+
 func Test_GetSearchRepository_TooManyResults(t *testing.T) {
 	transport := testTransport{
 		pathBodies: map[string]string{