Skip to content

Commit

Permalink
feat(appset): allow usage of project-scoped repos in git generators
Browse files Browse the repository at this point in the history 
Signed-off-by: Blake Pettersson <[email protected]>
  • Loading branch information
@blakepettersson
blakepettersson committed Dec 13, 2024
1 parent 87c853e commit 390fcca
Show file tree
Hide file tree
Showing 17 changed files with 840 additions and 727 deletions.
4 changes: 2 additions & 2 deletions applicationset/generators/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic

func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache, verifyCommit bool, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
// Directories, not files
allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, noRevisionCache, verifyCommit)
allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, appSetGenerator.Git.Project, noRevisionCache, verifyCommit)
if err != nil {
return nil, fmt.Errorf("error getting directories from repo: %w", err)
}
Expand All @@ -126,7 +126,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
// Get all files that match the requested path string, removing duplicates
allFiles := make(map[string][]byte)
for _, requestedPath := range appSetGenerator.Git.Files {
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path, noRevisionCache, verifyCommit)
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, appSetGenerator.Git.Project, requestedPath.Path, noRevisionCache, verifyCommit)
if err != nil {
return nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions applicationset/generators/git_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {

argoCDServiceMock := mocks.Repos{}

argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)

gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
applicationSetInfo := argoprojiov1alpha1.ApplicationSet{
Expand Down Expand Up @@ -622,7 +622,7 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {

argoCDServiceMock := mocks.Repos{}

argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)

gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
applicationSetInfo := argoprojiov1alpha1.ApplicationSet{
Expand Down Expand Up @@ -986,7 +986,7 @@ cluster:
t.Parallel()

argoCDServiceMock := mocks.Repos{}
argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)

gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
Expand Down Expand Up @@ -1342,7 +1342,7 @@ cluster:
t.Parallel()

argoCDServiceMock := mocks.Repos{}
argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)

gitGenerator := NewGitGenerator(&argoCDServiceMock, "")
Expand Down Expand Up @@ -1471,7 +1471,7 @@ func TestGitGenerator_GenerateParams(t *testing.T) {
argoCDServiceMock := mocks.Repos{}

if testCase.callGetDirectories {
argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCase.repoApps, testCase.repoPathsError)
argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCase.repoApps, testCase.repoPathsError)
}
gitGenerator := NewGitGenerator(&argoCDServiceMock, "namespace")

Expand Down
36 changes: 18 additions & 18 deletions applicationset/services/mocks/Repos.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

45 changes: 34 additions & 11 deletions applicationset/services/repo_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,21 @@ package services

import (
"context"
"errors"
"fmt"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
"github.com/argoproj/argo-cd/v2/reposerver/apiclient"
"github.com/argoproj/argo-cd/v2/util/git"
"github.com/argoproj/argo-cd/v2/util/io"
"github.com/argoproj/argo-cd/v2/util/repository"
)

type argoCDService struct {
getRepository func(ctx context.Context, url, project string) (*v1alpha1.Repository, error)
listRepositories func(ctx context.Context) ([]*v1alpha1.Repository, error)
storecreds git.CredsStore
submoduleEnabled bool
repoServerClientSet apiclient.Clientset
Expand All @@ -20,25 +25,34 @@ type argoCDService struct {

type Repos interface {
// GetFiles returns content of files (not directories) within the target repo
GetFiles(ctx context.Context, repoURL string, revision string, pattern string, noRevisionCache, verifyCommit bool) (map[string][]byte, error)
GetFiles(ctx context.Context, repoURL, revision, project, pattern string, noRevisionCache, verifyCommit bool) (map[string][]byte, error)

// GetDirectories returns a list of directories (not files) within the target repo
GetDirectories(ctx context.Context, repoURL string, revision string, noRevisionCache, verifyCommit bool) ([]string, error)
GetDirectories(ctx context.Context, repoURL, revision, project string, noRevisionCache, verifyCommit bool) ([]string, error)
}

func NewArgoCDService(getRepository func(ctx context.Context, url, project string) (*v1alpha1.Repository, error), submoduleEnabled bool, repoClientset apiclient.Clientset, newFileGlobbingEnabled bool) (Repos, error) {
func NewArgoCDService(listRepositories func(ctx context.Context) ([]*v1alpha1.Repository, error), submoduleEnabled bool, repoClientset apiclient.Clientset, newFileGlobbingEnabled bool) (Repos, error) {
return &argoCDService{
getRepository: getRepository,
listRepositories: listRepositories,
submoduleEnabled: submoduleEnabled,
repoServerClientSet: repoClientset,
newFileGlobbingEnabled: newFileGlobbingEnabled,
}, nil
}

func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision string, pattern string, noRevisionCache, verifyCommit bool) (map[string][]byte, error) {
repo, err := a.getRepository(ctx, repoURL, "")
func (a *argoCDService) GetFiles(ctx context.Context, repoURL, revision, project, pattern string, noRevisionCache, verifyCommit bool) (map[string][]byte, error) {
repos, err := a.listRepositories(ctx)
if err != nil {
return nil, fmt.Errorf("error in ListRepositories: %w", err)
}

repo, err := repository.FilterRepositoryByProjectAndURL(repos, repoURL, project)
if err != nil {
return nil, fmt.Errorf("error in GetRepository: %w", err)
if errors.Is(err, status.Error(codes.PermissionDenied, "permission denied")) {
repo = &v1alpha1.Repository{Repo: repoURL}
} else {
return nil, fmt.Errorf("error retrieving Git files: %w", err)
}
}

fileRequest := &apiclient.GitFilesRequest{
Expand All @@ -63,10 +77,19 @@ func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision s
return fileResponse.GetMap(), nil
}

func (a *argoCDService) GetDirectories(ctx context.Context, repoURL string, revision string, noRevisionCache, verifyCommit bool) ([]string, error) {
repo, err := a.getRepository(ctx, repoURL, "")
func (a *argoCDService) GetDirectories(ctx context.Context, repoURL, revision, project string, noRevisionCache, verifyCommit bool) ([]string, error) {
repos, err := a.listRepositories(ctx)
if err != nil {
return nil, fmt.Errorf("error in ListRepositories: %w", err)
}

repo, err := repository.FilterRepositoryByProjectAndURL(repos, repoURL, project)
if err != nil {
return nil, fmt.Errorf("error in GetRepository: %w", err)
if errors.Is(err, status.Error(codes.PermissionDenied, "permission denied")) {
repo = &v1alpha1.Repository{Repo: repoURL}
} else {
return nil, fmt.Errorf("error retrieving Git files: %w", err)
}
}

dirRequest := &apiclient.GitDirectoriesRequest{
Expand Down
57 changes: 33 additions & 24 deletions applicationset/services/repo_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@ func TestGetDirectories(t *testing.T) {
type fields struct {
storecreds git.CredsStore
submoduleEnabled bool
getRepository func(ctx context.Context, url, project string) (*v1alpha1.Repository, error)
listRepositories func(ctx context.Context) ([]*v1alpha1.Repository, error)
repoServerClientFuncs []func(*repo_mocks.RepoServerServiceClient)
}
type args struct {
ctx context.Context
repoURL string
revision string
project string
noRevisionCache bool
verifyCommit bool
}
Expand All @@ -38,13 +39,13 @@ func TestGetDirectories(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{name: "ErrorGettingRepos", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return nil, fmt.Errorf("unable to get repos")
},
}, args: args{}, want: nil, wantErr: assert.Error},
{name: "ErrorGettingDirs", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -53,8 +54,10 @@ func TestGetDirectories(t *testing.T) {
},
}, args: args{}, want: nil, wantErr: assert.Error},
{name: "HappyCase", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{
Repo: "foo",
}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -63,10 +66,12 @@ func TestGetDirectories(t *testing.T) {
}, nil)
},
},
}, args: args{}, want: []string{"foo", "foo/bar", "bar/foo"}, wantErr: assert.NoError},
}, args: args{
repoURL: "foo",
}, want: []string{"foo", "foo/bar", "bar/foo"}, wantErr: assert.NoError},
{name: "ErrorVerifyingCommit", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -84,12 +89,12 @@ func TestGetDirectories(t *testing.T) {
}

a := &argoCDService{
getRepository: tt.fields.getRepository,
listRepositories: tt.fields.listRepositories,
storecreds: tt.fields.storecreds,
submoduleEnabled: tt.fields.submoduleEnabled,
repoServerClientSet: &repo_mocks.Clientset{RepoServerServiceClient: mockRepoClient},
}
got, err := a.GetDirectories(tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.noRevisionCache, tt.args.verifyCommit)
got, err := a.GetDirectories(tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.project, tt.args.noRevisionCache, tt.args.verifyCommit)
if !tt.wantErr(t, err, fmt.Sprintf("GetDirectories(%v, %v, %v, %v)", tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.noRevisionCache)) {
return
}
Expand All @@ -103,7 +108,7 @@ func TestGetFiles(t *testing.T) {
storecreds git.CredsStore
submoduleEnabled bool
repoServerClientFuncs []func(*repo_mocks.RepoServerServiceClient)
getRepository func(ctx context.Context, url, project string) (*v1alpha1.Repository, error)
listRepositories func(ctx context.Context) ([]*v1alpha1.Repository, error)
}
type args struct {
ctx context.Context
Expand All @@ -121,13 +126,13 @@ func TestGetFiles(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{name: "ErrorGettingRepos", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return nil, fmt.Errorf("unable to get repos")
},
}, args: args{}, want: nil, wantErr: assert.Error},
{name: "ErrorGettingFiles", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -136,8 +141,10 @@ func TestGetFiles(t *testing.T) {
},
}, args: args{}, want: nil, wantErr: assert.Error},
{name: "HappyCase", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{
Repo: "foo",
}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -149,13 +156,15 @@ func TestGetFiles(t *testing.T) {
}, nil)
},
},
}, args: args{}, want: map[string][]byte{
}, args: args{
repoURL: "foo",
}, want: map[string][]byte{
"foo.json": []byte("hello: world!"),
"bar.yaml": []byte("yay: appsets"),
}, wantErr: assert.NoError},
{name: "ErrorVerifyingCommit", fields: fields{
getRepository: func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
listRepositories: func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{}}, nil
},
repoServerClientFuncs: []func(*repo_mocks.RepoServerServiceClient){
func(client *repo_mocks.RepoServerServiceClient) {
Expand All @@ -173,12 +182,12 @@ func TestGetFiles(t *testing.T) {
}

a := &argoCDService{
getRepository: tt.fields.getRepository,
listRepositories: tt.fields.listRepositories,
storecreds: tt.fields.storecreds,
submoduleEnabled: tt.fields.submoduleEnabled,
repoServerClientSet: &repo_mocks.Clientset{RepoServerServiceClient: mockRepoClient},
}
got, err := a.GetFiles(tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.pattern, tt.args.noRevisionCache, tt.args.verifyCommit)
got, err := a.GetFiles(tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.pattern, "", tt.args.noRevisionCache, tt.args.verifyCommit)
if !tt.wantErr(t, err, fmt.Sprintf("GetFiles(%v, %v, %v, %v, %v)", tt.args.ctx, tt.args.repoURL, tt.args.revision, tt.args.pattern, tt.args.noRevisionCache)) {
return
}
Expand All @@ -188,8 +197,8 @@ func TestGetFiles(t *testing.T) {
}

func TestNewArgoCDService(t *testing.T) {
service, err := NewArgoCDService(func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) {
return &v1alpha1.Repository{}, nil
service, err := NewArgoCDService(func(ctx context.Context) ([]*v1alpha1.Repository, error) {
return []*v1alpha1.Repository{{}}, nil
}, false, &repo_mocks.Clientset{}, false)
require.NoError(t, err)
assert.NotNil(t, service)
Expand Down
Loading

0 comments on commit 390fcca

Please sign in to comment.