From d8b75f4ed173c2389dcb3dd89a4213fd8ce78f42 Mon Sep 17 00:00:00 2001 From: James Telfer <792299+jamestelfer@users.noreply.github.com> Date: Fri, 10 Nov 2023 06:30:16 +1100 Subject: [PATCH] feat(github): use code search to select repositories (#399) Co-authored-by: Johan Lindell --- cmd/platform.go | 9 +- go.mod | 14 +- go.sum | 36 +++--- internal/scm/github/github.go | 79 ++++++++++++ internal/scm/github/github_test.go | 199 +++++++++++++++++++++++++---- 5 files changed, 287 insertions(+), 50 deletions(-) diff --git a/cmd/platform.go b/cmd/platform.go index e63824d1..fb64cd34 100644 --- a/cmd/platform.go +++ b/cmd/platform.go @@ -28,7 +28,8 @@ func configurePlatform(cmd *cobra.Command) { flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.") flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.") flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".") - flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories") + flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories.") + flags.StringP("code-search", "", "", "Use a code search to find a set of repositories to target (GitHub only). Repeated results from a given repository will be ignored, forks are NOT included by default (use `fork:true` to include them). See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-code.") flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.") flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".") flags.BoolP("include-subgroups", "", false, "Include GitLab subgroups when using the --group flag.") @@ -124,14 +125,15 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu users, _ := flag.GetStringSlice("user") repos, _ := flag.GetStringSlice("repo") repoSearch, _ := flag.GetString("repo-search") + codeSearch, _ := flag.GetString("code-search") topics, _ := flag.GetStringSlice("topic") forkMode, _ := flag.GetBool("fork") forkOwner, _ := flag.GetString("fork-owner") sshAuth, _ := flag.GetBool("ssh-auth") skipForks, _ := flag.GetBool("skip-forks") - if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" { - return nil, errors.New("no organization, user, repo or repo-search set") + if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" && codeSearch == "" { + return nil, errors.New("no organization, user, repo, repo-search or code-search set") } token, err := getToken(flag) @@ -166,6 +168,7 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu BaseURL: gitBaseURL, TransportMiddleware: http.NewLoggingRoundTripper, RepoListing: github.RepositoryListing{ + CodeSearch: codeSearch, Organizations: orgs, Users: users, Repositories: repoRefs, diff --git a/go.mod b/go.mod index a78d4d8d..c1a09da3 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 github.com/xanzy/go-gitlab v0.93.2 + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 golang.org/x/oauth2 v0.13.0 ) @@ -60,14 +61,13 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 48e23ad6..32115085 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -321,8 +321,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -348,8 +348,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -387,8 +387,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -412,8 +412,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -459,15 +459,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -479,8 +479,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -535,8 +535,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/scm/github/github.go b/internal/scm/github/github.go index 10a1c6f7..12a34d1b 100755 --- a/internal/scm/github/github.go +++ b/internal/scm/github/github.go @@ -13,6 +13,7 @@ import ( "github.com/lindell/multi-gitter/internal/scm" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" "golang.org/x/oauth2" ) @@ -109,6 +110,7 @@ type RepositoryListing struct { Users []string Repositories []RepositoryReference RepositorySearch string + CodeSearch string Topics []string SkipForks bool } @@ -222,6 +224,14 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err allRepos = append(allRepos, repos...) } + if g.CodeSearch != "" { + repos, err := g.getCodeSearchRepositories(ctx, g.CodeSearch) + if err != nil { + return nil, errors.Wrapf(err, "could not get code search results for '%s'", g.CodeSearch) + } + allRepos = append(allRepos, repos...) + } + // Remove duplicate repos repoMap := map[string]*github.Repository{} for _, repo := range allRepos { @@ -333,6 +343,75 @@ func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*g return repos, nil } +func (g *Github) getCodeSearchRepositories(ctx context.Context, search string) ([]*github.Repository, error) { + resultRepos := make(map[string]RepositoryReference) + + i := 1 + for { + rr, _, err := retry(ctx, func() ([]*github.CodeResult, *github.Response, error) { + rr, resp, err := g.ghClient.Search.Code(ctx, search, &github.SearchOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: 100, + }, + }) + + if err != nil { + return nil, nil, err + } + + if rr.GetIncompleteResults() { + // can occur when search times out on the server: for now, fail instead + // of handling the issue + return nil, nil, fmt.Errorf("search timed out on GitHub and was marked incomplete: try refining the search to return fewer results or be less complex") + } + + if rr.GetTotal() > 1000 { + return nil, nil, fmt.Errorf("%d results for this search, but only the first 1000 results will be returned: try refining your search terms", rr.GetTotal()) + } + + return rr.CodeResults, resp, nil + }) + + if err != nil { + return nil, err + } + + for _, r := range rr { + repo := r.Repository + + resultRepos[repo.GetFullName()] = RepositoryReference{ + OwnerName: repo.GetOwner().GetLogin(), + Name: repo.GetName(), + } + } + + if len(rr) != 100 { + break + } + i++ + } + + // Code search does not return full details (like permissions). So for each + // repo discovered, we have to query it again. + repoNames := maps.Values(resultRepos) + return g.getAllRepositories(ctx, repoNames) +} + +func (g *Github) getAllRepositories(ctx context.Context, repoRefs []RepositoryReference) ([]*github.Repository, error) { + var repos []*github.Repository + + for _, ref := range repoRefs { + r, err := g.getRepository(ctx, ref) + if err != nil { + return nil, err + } + repos = append(repos, r) + } + + return repos, nil +} + func (g *Github) getRepository(ctx context.Context, repoRef RepositoryReference) (*github.Repository, error) { repo, _, err := retry(ctx, func() (*github.Repository, *github.Response, error) { return g.ghClient.Repositories.Get(ctx, repoRef.OwnerName, repoRef.Name) diff --git a/internal/scm/github/github_test.go b/internal/scm/github/github_test.go index 0be69fc4..6070e234 100644 --- a/internal/scm/github/github_test.go +++ b/internal/scm/github/github_test.go @@ -152,6 +152,55 @@ func Test_GetRepositories(t *testing.T) { } ] }`, + "/search/code": `{ + "total_count": 1, + "incomplete_results": false, + "items": [ + { + "name": "classes.js", + "path": "src/attributes/classes.js", + "sha": "d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "url": "https://api.github.com/repositories/167174/contents/src/attributes/classes.js?ref=825ac3773694e0cd23ee74895fd5aeb535b27da4", + "git_url": "https://api.github.com/repositories/167174/git/blobs/d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "html_url": "https://github.com/jquery/jquery/blob/825ac3773694e0cd23ee74895fd5aeb535b27da4/src/attributes/classes.js", + "repository": { + "name": "search-code1", + "full_name": "test-org/search-code1", + "owner": { + "login": "test-org", + "type": "Organization", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/test-org/search-code1", + "fork": false + }, + "score": 1 + } + ] + }`, + "/repos/test-org/search-code1": `{ + "id": 2, + "name": "search-code1", + "full_name": "test-org/search-code1", + "private": false, + "owner": { + "login": "test-org", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/test-org/search-code1", + "fork": false, + "archived": false, + "disabled": false, + "default_branch": "main", + "permissions": { + "admin": true, + "push": true, + "pull": true + }, + "created_at": "2020-01-02T16:49:17Z" + }`, }, } @@ -256,6 +305,25 @@ func Test_GetRepositories(t *testing.T) { } } + // Code search + { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + CodeSearch: "search-string", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) + + repos, err := gh.GetRepositories(context.Background()) + assert.NoError(t, err) + if assert.Len(t, repos, 1) { + assert.Equal(t, "main", repos[0].DefaultBranch()) + assert.Equal(t, "test-org/search-code1", repos[0].FullName()) + } + } + // Multiple { gh, err := github.New(github.Config{ @@ -313,7 +381,7 @@ func Test_GetRepositories(t *testing.T) { } } -func Test_GetSearchRepository_Incomplete(t *testing.T) { +func Test_SearchIncomplete(t *testing.T) { transport := testTransport{ pathBodies: map[string]string{ "/search/repositories": `{ @@ -348,24 +416,68 @@ func Test_GetSearchRepository_Incomplete(t *testing.T) { } ] }`, + "/search/code": `{ + "total_count": 1, + "incomplete_results": true, + "items": [ + { + "name": "classes.js", + "path": "src/attributes/classes.js", + "sha": "d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "url": "https://api.github.com/repositories/167174/contents/src/attributes/classes.js?ref=825ac3773694e0cd23ee74895fd5aeb535b27da4", + "git_url": "https://api.github.com/repositories/167174/git/blobs/d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "html_url": "https://github.com/jquery/jquery/blob/825ac3773694e0cd23ee74895fd5aeb535b27da4/src/attributes/classes.js", + "repository": { + "name": "search-code1", + "full_name": "test-org/search-code1", + "owner": { + "login": "test-org", + "type": "Organization", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/test-org/search-code1", + "fork": false + }, + "score": 1 + } + ] + }`, }, } - gh, err := github.New(github.Config{ - TransportMiddleware: transport.Wrapper, - RepoListing: github.RepositoryListing{ - RepositorySearch: "search-string", - }, - MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + t.Run("getSearchRepositories", func(t *testing.T) { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + RepositorySearch: "search-string", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) + + repos, err := gh.GetRepositories(context.Background()) + assert.ErrorContains(t, err, "search timed out on GitHub and was marked incomplete") + assert.Len(t, repos, 0) }) - require.NoError(t, err) - repos, err := gh.GetRepositories(context.Background()) - assert.ErrorContains(t, err, "search timed out on GitHub and was marked incomplete") - assert.Len(t, repos, 0) + t.Run("getCodeSearchRepositories", func(t *testing.T) { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + CodeSearch: "search-string", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) + + repos, err := gh.GetRepositories(context.Background()) + assert.ErrorContains(t, err, "search timed out on GitHub and was marked incomplete") + assert.Len(t, repos, 0) + }) } -func Test_GetSearchRepository_TooManyResults(t *testing.T) { +func Test_SearchTooManyResults(t *testing.T) { transport := testTransport{ pathBodies: map[string]string{ "/search/repositories": `{ @@ -400,19 +512,62 @@ func Test_GetSearchRepository_TooManyResults(t *testing.T) { } ] }`, + "/search/code": `{ + "total_count": 1532, + "incomplete_results": false, + "items": [ + { + "name": "classes.js", + "path": "src/attributes/classes.js", + "sha": "d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "url": "https://api.github.com/repositories/167174/contents/src/attributes/classes.js?ref=825ac3773694e0cd23ee74895fd5aeb535b27da4", + "git_url": "https://api.github.com/repositories/167174/git/blobs/d7212f9dee2dcc18f084d7df8f417b80846ded5a", + "html_url": "https://github.com/jquery/jquery/blob/825ac3773694e0cd23ee74895fd5aeb535b27da4/src/attributes/classes.js", + "repository": { + "name": "search-code1", + "full_name": "test-org/search-code1", + "owner": { + "login": "test-org", + "type": "Organization", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/test-org/search-code1", + "fork": false + }, + "score": 1 + } + ] + }`, }, } + t.Run("getSearchRepositories", func(t *testing.T) { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + RepositorySearch: "search-string", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) - gh, err := github.New(github.Config{ - TransportMiddleware: transport.Wrapper, - RepoListing: github.RepositoryListing{ - RepositorySearch: "search-string", - }, - MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + repos, err := gh.GetRepositories(context.Background()) + assert.ErrorContains(t, err, "only the first 1000 results will be returned") + assert.Len(t, repos, 0) }) - require.NoError(t, err) - repos, err := gh.GetRepositories(context.Background()) - assert.ErrorContains(t, err, "only the first 1000 results will be returned") - assert.Len(t, repos, 0) + t.Run("getCodeSearchRepositories", func(t *testing.T) { + gh, err := github.New(github.Config{ + TransportMiddleware: transport.Wrapper, + RepoListing: github.RepositoryListing{ + CodeSearch: "search-string", + }, + MergeTypes: []scm.MergeType{scm.MergeTypeMerge}, + }) + require.NoError(t, err) + + repos, err := gh.GetRepositories(context.Background()) + assert.ErrorContains(t, err, "only the first 1000 results will be returned") + assert.Len(t, repos, 0) + }) }