diff --git a/acorns/config/customconfigint.go b/acorns/config/customconfigint.go index 75d5b54..3f9939b 100644 --- a/acorns/config/customconfigint.go +++ b/acorns/config/customconfigint.go @@ -35,6 +35,7 @@ type CustomConfiguration interface { AuthOidcTokenAudience() string AuthGroupWrite() string + MetadataRepoUrl() string MetadataRepoMainline() string UpdateJobIntervalCronPart() string @@ -91,6 +92,7 @@ const ( KeyAuthOidcKeySetUrl = "AUTH_OIDC_KEY_SET_URL" KeyAuthOidcTokenAudience = "AUTH_OIDC_TOKEN_AUDIENCE" KeyAuthGroupWrite = "AUTH_GROUP_WRITE" + KeyMetadataRepoUrl = "METADATA_REPO_URL" KeyMetadataRepoMainline = "METADATA_REPO_MAINLINE" KeyUpdateJobIntervalMinutes = "UPDATE_JOB_INTERVAL_MINUTES" KeyUpdateJobTimeoutSeconds = "UPDATE_JOB_TIMEOUT_SECONDS" diff --git a/docs/local-config.template.yaml b/docs/local-config.template.yaml index 1b991a0..70362db 100644 --- a/docs/local-config.template.yaml +++ b/docs/local-config.template.yaml @@ -32,6 +32,7 @@ VAULT_SECRETS_CONFIG: >- AUTH_OIDC_KEY_SET_URL: https://login.microsoftonline.com//discovery/v2.0/keys AUTH_OIDC_TOKEN_AUDIENCE: +METADATA_REPO_URL: https://github.com/Interhyp/service-metadata-example SSH_METADATA_REPO_URL: ssh://git@github.com/Interhyp/service-metadata-example.git UPDATE_JOB_INTERVAL_MINUTES: 15 diff --git a/internal/repository/config/accessors.go b/internal/repository/config/accessors.go index bd4caf5..f2d8ffc 100644 --- a/internal/repository/config/accessors.go +++ b/internal/repository/config/accessors.go @@ -90,6 +90,10 @@ func (c *CustomConfigImpl) KafkaGroupIdOverride() string { return c.VKafkaGroupIdOverride } +func (c *CustomConfigImpl) MetadataRepoUrl() string { + return c.VMetadataRepoUrl +} + func (c *CustomConfigImpl) MetadataRepoMainline() string { return c.VMetadataRepoMainline } diff --git a/internal/repository/config/config.go b/internal/repository/config/config.go index 6c5f10f..cb2d2aa 100644 --- a/internal/repository/config/config.go +++ b/internal/repository/config/config.go @@ -156,6 +156,13 @@ var CustomConfigItems = []auconfigapi.ConfigItem{ Description: "group name or id for write access to this service", Validate: auconfigapi.ConfigNeedsNoValidation, }, + { + Key: config.KeyMetadataRepoUrl, + EnvName: config.KeyMetadataRepoUrl, + Default: "", + Description: "git clone url for service-metadata repository", + Validate: auconfigenv.ObtainNotEmptyValidator(), + }, { Key: config.KeyMetadataRepoMainline, EnvName: config.KeyMetadataRepoMainline, diff --git a/internal/repository/config/plumbing.go b/internal/repository/config/plumbing.go index ccab2bb..6789e8d 100644 --- a/internal/repository/config/plumbing.go +++ b/internal/repository/config/plumbing.go @@ -34,6 +34,7 @@ type CustomConfigImpl struct { VAuthOidcTokenAudience string VAuthGroupWrite string VKafkaGroupIdOverride string + VMetadataRepoUrl string VMetadataRepoMainline string VUpdateJobIntervalCronPart string VUpdateJobTimeoutSeconds uint16 @@ -86,6 +87,7 @@ func (c *CustomConfigImpl) Obtain(getter func(key string) string) { c.VAuthOidcKeySetUrl = getter(config.KeyAuthOidcKeySetUrl) c.VAuthOidcTokenAudience = getter(config.KeyAuthOidcTokenAudience) c.VAuthGroupWrite = getter(config.KeyAuthGroupWrite) + c.VMetadataRepoUrl = getter(config.KeyMetadataRepoUrl) c.VMetadataRepoMainline = getter(config.KeyMetadataRepoMainline) c.VUpdateJobIntervalCronPart = getter(config.KeyUpdateJobIntervalMinutes) c.VUpdateJobTimeoutSeconds = toUint16(getter(config.KeyUpdateJobTimeoutSeconds)) diff --git a/internal/repository/config/validation_test.go b/internal/repository/config/validation_test.go index 04ee5e9..c54de59 100644 --- a/internal/repository/config/validation_test.go +++ b/internal/repository/config/validation_test.go @@ -73,7 +73,7 @@ func TestValidate_LotsOfErrors(t *testing.T) { _, err := tstSetupCutAndLogRecorder(t, "invalid-config-values.yaml") require.NotNil(t, err) - require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 26 error(s). See details above") + require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 27 error(s). See details above") actualLog := goauzerolog.RecordedLogForTesting.String() @@ -128,6 +128,7 @@ func TestAccessors(t *testing.T) { require.Equal(t, "http://keyset", config.Custom(cut).AuthOidcKeySetUrl()) require.Equal(t, "some-audience", config.Custom(cut).AuthOidcTokenAudience()) require.Equal(t, "admin", config.Custom(cut).AuthGroupWrite()) + require.Equal(t, "http://metadata", config.Custom(cut).MetadataRepoUrl()) require.Equal(t, "git://metadata", config.Custom(cut).SSHMetadataRepositoryUrl()) require.Equal(t, "5", config.Custom(cut).UpdateJobIntervalCronPart()) require.Equal(t, uint16(30), config.Custom(cut).UpdateJobTimeoutSeconds()) diff --git a/internal/repository/metadata/acorn.go b/internal/repository/metadata/acorn.go index 981e0b9..fcb3327 100644 --- a/internal/repository/metadata/acorn.go +++ b/internal/repository/metadata/acorn.go @@ -62,7 +62,7 @@ func (r *Impl) SetupAcorn(registry auacornapi.AcornRegistry) error { return nil } -func (r *Impl) TeardownAcorn(_ auacornapi.AcornRegistry) error { +func (r *Impl) TeardownAcorn(registry auacornapi.AcornRegistry) error { ctx := auzerolog.AddLoggerToCtx(context.Background()) r.Discard(ctx) return nil diff --git a/internal/repository/metadata/metadata.go b/internal/repository/metadata/metadata.go index f592cd8..7b8ffa7 100644 --- a/internal/repository/metadata/metadata.go +++ b/internal/repository/metadata/metadata.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/go-git/go-git/v5/plumbing/transport/http" "io" "os" "strings" @@ -52,6 +53,10 @@ type Impl struct { consoleOutput bytes.Buffer } +const insecureSkipTLS = false + +var UseHTTP = true + func (r *Impl) pathsTouchedInCommit(ctx context.Context, commit *object.Commit) ([]string, error) { result := make([]string, 0) @@ -193,15 +198,31 @@ func (r *Impl) Clone(ctx context.Context) error { return err } - repo, err := git.CloneContext(childCtxWithTimeout, memory.NewStorage(), memfs.New(), &git.CloneOptions{ + cloneOpts := git.CloneOptions{ Auth: sshAuth, NoCheckout: false, Progress: r, // implements io.Writer, sends to Debug logging URL: r.CustomConfiguration.SSHMetadataRepositoryUrl(), ReferenceName: plumbing.ReferenceName(r.CustomConfiguration.MetadataRepoMainline()), - }) + } + + if UseHTTP { + cloneOpts.Auth = &http.BasicAuth{ + Username: r.CustomConfiguration.BitbucketUsername(), + Password: r.CustomConfiguration.BitbucketPassword(), + } + cloneOpts.URL = r.CustomConfiguration.MetadataRepoUrl() + cloneOpts.InsecureSkipTLS = insecureSkipTLS + } + + repo, err := git.CloneContext(childCtxWithTimeout, memory.NewStorage(), memfs.New(), &cloneOpts) if err != nil { r.Logging.Logger().Ctx(ctx).Warn().Print("git clone failed - console output was: ", r.sanitizedConsoleOutput()) + r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git clone failed - returned error: %s", err.Error()) + + r.logContextErrorDetails(childCtxWithTimeout, "git clone", "childCtxWithTimeout") + r.logContextErrorDetails(ctx, "git clone", "ctx") + return err } r.GitRepo = repo @@ -237,14 +258,29 @@ func (r *Impl) Pull(ctx context.Context) error { return err } - err = tree.PullContext(childCtxWithTimeout, &git.PullOptions{ + pullOpts := git.PullOptions{ Auth: sshAuth, Progress: r, // implements io.Writer, sends to Debug logging RemoteName: "origin", ReferenceName: plumbing.ReferenceName(r.CustomConfiguration.MetadataRepoMainline()), - }) + } + + if UseHTTP { + pullOpts.Auth = &http.BasicAuth{ + Username: r.CustomConfiguration.BitbucketUsername(), + Password: r.CustomConfiguration.BitbucketPassword(), + } + pullOpts.InsecureSkipTLS = insecureSkipTLS + } + + err = tree.PullContext(childCtxWithTimeout, &pullOpts) if err != nil && err != git.NoErrAlreadyUpToDate { r.Logging.Logger().Ctx(ctx).Warn().Print("git pull failed - console output was: ", r.sanitizedConsoleOutput()) + r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git pull failed - returned error: %s", err.Error()) + + r.logContextErrorDetails(childCtxWithTimeout, "git pull", "childCtxWithTimeout") + r.logContextErrorDetails(ctx, "git pull", "ctx") + return err } @@ -327,13 +363,28 @@ func (r *Impl) Push(ctx context.Context) error { return err } - err = r.GitRepo.PushContext(childCtxWithTimeout, &git.PushOptions{ + pushOpts := git.PushOptions{ Auth: sshAuth, Progress: r, // implements io.Writer, sends to Debug logging RemoteName: "origin", - }) + } + + if UseHTTP { + pushOpts.Auth = &http.BasicAuth{ + Username: r.CustomConfiguration.BitbucketUsername(), + Password: r.CustomConfiguration.BitbucketPassword(), + } + pushOpts.InsecureSkipTLS = insecureSkipTLS + } + + err = r.GitRepo.PushContext(childCtxWithTimeout, &pushOpts) if err != nil && err != git.NoErrAlreadyUpToDate { r.Logging.Logger().Ctx(ctx).Warn().Print("git push failed - console output was: ", r.sanitizedConsoleOutput()) + r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git push failed - returned error: %s", err.Error()) + + r.logContextErrorDetails(childCtxWithTimeout, "git push", "childCtxWithTimeout") + r.logContextErrorDetails(ctx, "git push", "ctx") + return err } @@ -356,6 +407,15 @@ func (r *Impl) Discard(ctx context.Context) { r.KnownCommits = make(map[string]bool) } +func (r *Impl) logContextErrorDetails(ctx context.Context, operation string, contextName string) { + ctxCause := context.Cause(ctx) + if ctxCause != nil { + r.Logging.Logger().Ctx(ctx).Warn().WithErr(ctxCause).Printf("%s failed - %s cancellation cause: %s", operation, contextName, ctxCause.Error()) + } else { + r.Logging.Logger().Ctx(ctx).Warn().Printf("%s failed - %s not cancelled", operation, contextName) + } +} + func (r *Impl) LastUpdated() time.Time { r.mu.Lock() defer r.mu.Unlock() diff --git a/test/acceptance/configmock/configmock.go b/test/acceptance/configmock/configmock.go index 97abaaa..5d344ea 100644 --- a/test/acceptance/configmock/configmock.go +++ b/test/acceptance/configmock/configmock.go @@ -78,6 +78,11 @@ func (c *MockConfig) SSHMetadataRepositoryUrl() string { panic("implement me") } +func (c *MockConfig) MetadataRepoUrl() string { + //TODO implement me + panic("implement me") +} + func (c *MockConfig) BitbucketUsername() string { //TODO implement me panic("implement me") diff --git a/test/resources/valid-config-unique.yaml b/test/resources/valid-config-unique.yaml index e859ded..44ba9a1 100644 --- a/test/resources/valid-config-unique.yaml +++ b/test/resources/valid-config-unique.yaml @@ -24,6 +24,7 @@ AUTH_GROUP_WRITE: admin SSH_PRIVATE_KEY: some-private-key SSH_METADATA_REPO_URL: git://metadata +METADATA_REPO_URL: http://metadata UPDATE_JOB_INTERVAL_MINUTES: 5 UPDATE_JOB_TIMEOUT_SECONDS: 30 diff --git a/test/resources/valid-config.yaml b/test/resources/valid-config.yaml index 47e0db1..4b7883b 100644 --- a/test/resources/valid-config.yaml +++ b/test/resources/valid-config.yaml @@ -9,6 +9,7 @@ AUTH_OIDC_TOKEN_AUDIENCE: some-audience AUTH_GROUP_WRITE: admin SSH_METADATA_REPO_URL: git://metadata +METADATA_REPO_URL: http://metadata UPDATE_JOB_INTERVAL_MINUTES: 5