Merge pull request #145 from Interhyp/RELTEC-11181-metadata-debugging…

…-push

Reltec 11181 metadata debugging push

acorns/config/customconfigint.go

@@ -35,6 +35,7 @@ type CustomConfiguration interface {
 	AuthOidcTokenAudience() string
 	AuthGroupWrite() string
 
+	MetadataRepoUrl() string
 	MetadataRepoMainline() string
 
 	UpdateJobIntervalCronPart() string
@@ -91,6 +92,7 @@ const (
 	KeyAuthOidcKeySetUrl              = "AUTH_OIDC_KEY_SET_URL"
 	KeyAuthOidcTokenAudience          = "AUTH_OIDC_TOKEN_AUDIENCE"
 	KeyAuthGroupWrite                 = "AUTH_GROUP_WRITE"
+	KeyMetadataRepoUrl                = "METADATA_REPO_URL"
 	KeyMetadataRepoMainline           = "METADATA_REPO_MAINLINE"
 	KeyUpdateJobIntervalMinutes       = "UPDATE_JOB_INTERVAL_MINUTES"
 	KeyUpdateJobTimeoutSeconds        = "UPDATE_JOB_TIMEOUT_SECONDS"

docs/local-config.template.yaml

@@ -32,6 +32,7 @@ VAULT_SECRETS_CONFIG: >-
 AUTH_OIDC_KEY_SET_URL: https://login.microsoftonline.com/<YOU MUST ADD CLIENT ID HERE>/discovery/v2.0/keys
 AUTH_OIDC_TOKEN_AUDIENCE: <YOU MUST ADD TOKEN AUDIENCE HERE>
 
+METADATA_REPO_URL: https://github.com/Interhyp/service-metadata-example
 SSH_METADATA_REPO_URL: ssh://[email protected]/Interhyp/service-metadata-example.git
 
 UPDATE_JOB_INTERVAL_MINUTES: 15

internal/repository/config/accessors.go

@@ -90,6 +90,10 @@ func (c *CustomConfigImpl) KafkaGroupIdOverride() string {
 	return c.VKafkaGroupIdOverride
 }
 
+func (c *CustomConfigImpl) MetadataRepoUrl() string {
+	return c.VMetadataRepoUrl
+}
+
 func (c *CustomConfigImpl) MetadataRepoMainline() string {
 	return c.VMetadataRepoMainline
 }

internal/repository/config/config.go

@@ -156,6 +156,13 @@ var CustomConfigItems = []auconfigapi.ConfigItem{
 		Description: "group name or id for write access to this service",
 		Validate:    auconfigapi.ConfigNeedsNoValidation,
 	},
+	{
+		Key:         config.KeyMetadataRepoUrl,
+		EnvName:     config.KeyMetadataRepoUrl,
+		Default:     "",
+		Description: "git clone url for service-metadata repository",
+		Validate:    auconfigenv.ObtainNotEmptyValidator(),
+	},
 	{
 		Key:         config.KeyMetadataRepoMainline,
 		EnvName:     config.KeyMetadataRepoMainline,

internal/repository/config/plumbing.go

@@ -34,6 +34,7 @@ type CustomConfigImpl struct {
 	VAuthOidcTokenAudience          string
 	VAuthGroupWrite                 string
 	VKafkaGroupIdOverride           string
+	VMetadataRepoUrl                string
 	VMetadataRepoMainline           string
 	VUpdateJobIntervalCronPart      string
 	VUpdateJobTimeoutSeconds        uint16
@@ -86,6 +87,7 @@ func (c *CustomConfigImpl) Obtain(getter func(key string) string) {
 	c.VAuthOidcKeySetUrl = getter(config.KeyAuthOidcKeySetUrl)
 	c.VAuthOidcTokenAudience = getter(config.KeyAuthOidcTokenAudience)
 	c.VAuthGroupWrite = getter(config.KeyAuthGroupWrite)
+	c.VMetadataRepoUrl = getter(config.KeyMetadataRepoUrl)
 	c.VMetadataRepoMainline = getter(config.KeyMetadataRepoMainline)
 	c.VUpdateJobIntervalCronPart = getter(config.KeyUpdateJobIntervalMinutes)
 	c.VUpdateJobTimeoutSeconds = toUint16(getter(config.KeyUpdateJobTimeoutSeconds))

internal/repository/config/validation_test.go

@@ -73,7 +73,7 @@ func TestValidate_LotsOfErrors(t *testing.T) {
 	_, err := tstSetupCutAndLogRecorder(t, "invalid-config-values.yaml")
 
 	require.NotNil(t, err)
-	require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 26 error(s). See details above")
+	require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 27 error(s). See details above")
 
 	actualLog := goauzerolog.RecordedLogForTesting.String()
 
@@ -128,6 +128,7 @@ func TestAccessors(t *testing.T) {
 	require.Equal(t, "http://keyset", config.Custom(cut).AuthOidcKeySetUrl())
 	require.Equal(t, "some-audience", config.Custom(cut).AuthOidcTokenAudience())
 	require.Equal(t, "admin", config.Custom(cut).AuthGroupWrite())
+	require.Equal(t, "http://metadata", config.Custom(cut).MetadataRepoUrl())
 	require.Equal(t, "git://metadata", config.Custom(cut).SSHMetadataRepositoryUrl())
 	require.Equal(t, "5", config.Custom(cut).UpdateJobIntervalCronPart())
 	require.Equal(t, uint16(30), config.Custom(cut).UpdateJobTimeoutSeconds())

internal/repository/metadata/acorn.go

@@ -62,7 +62,7 @@ func (r *Impl) SetupAcorn(registry auacornapi.AcornRegistry) error {
 	return nil
 }
 
-func (r *Impl) TeardownAcorn(_ auacornapi.AcornRegistry) error {
+func (r *Impl) TeardownAcorn(registry auacornapi.AcornRegistry) error {
 	ctx := auzerolog.AddLoggerToCtx(context.Background())
 	r.Discard(ctx)
 	return nil

internal/repository/metadata/metadata.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"io"
 	"os"
 	"strings"
@@ -52,6 +53,10 @@ type Impl struct {
 	consoleOutput bytes.Buffer
 }
 
+const insecureSkipTLS = false
+
+var UseHTTP = true
+
 func (r *Impl) pathsTouchedInCommit(ctx context.Context, commit *object.Commit) ([]string, error) {
 	result := make([]string, 0)
 
@@ -193,15 +198,31 @@ func (r *Impl) Clone(ctx context.Context) error {
 		return err
 	}
 
-	repo, err := git.CloneContext(childCtxWithTimeout, memory.NewStorage(), memfs.New(), &git.CloneOptions{
+	cloneOpts := git.CloneOptions{
 		Auth:          sshAuth,
 		NoCheckout:    false,
 		Progress:      r, // implements io.Writer, sends to Debug logging
 		URL:           r.CustomConfiguration.SSHMetadataRepositoryUrl(),
 		ReferenceName: plumbing.ReferenceName(r.CustomConfiguration.MetadataRepoMainline()),
-	})
+	}
+
+	if UseHTTP {
+		cloneOpts.Auth = &http.BasicAuth{
+			Username: r.CustomConfiguration.BitbucketUsername(),
+			Password: r.CustomConfiguration.BitbucketPassword(),
+		}
+		cloneOpts.URL = r.CustomConfiguration.MetadataRepoUrl()
+		cloneOpts.InsecureSkipTLS = insecureSkipTLS
+	}
+
+	repo, err := git.CloneContext(childCtxWithTimeout, memory.NewStorage(), memfs.New(), &cloneOpts)
 	if err != nil {
 		r.Logging.Logger().Ctx(ctx).Warn().Print("git clone failed - console output was: ", r.sanitizedConsoleOutput())
+		r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git clone failed - returned error: %s", err.Error())
+
+		r.logContextErrorDetails(childCtxWithTimeout, "git clone", "childCtxWithTimeout")
+		r.logContextErrorDetails(ctx, "git clone", "ctx")
+
 		return err
 	}
 	r.GitRepo = repo
@@ -237,14 +258,29 @@ func (r *Impl) Pull(ctx context.Context) error {
 		return err
 	}
 
-	err = tree.PullContext(childCtxWithTimeout, &git.PullOptions{
+	pullOpts := git.PullOptions{
 		Auth:          sshAuth,
 		Progress:      r, // implements io.Writer, sends to Debug logging
 		RemoteName:    "origin",
 		ReferenceName: plumbing.ReferenceName(r.CustomConfiguration.MetadataRepoMainline()),
-	})
+	}
+
+	if UseHTTP {
+		pullOpts.Auth = &http.BasicAuth{
+			Username: r.CustomConfiguration.BitbucketUsername(),
+			Password: r.CustomConfiguration.BitbucketPassword(),
+		}
+		pullOpts.InsecureSkipTLS = insecureSkipTLS
+	}
+
+	err = tree.PullContext(childCtxWithTimeout, &pullOpts)
 	if err != nil && err != git.NoErrAlreadyUpToDate {
 		r.Logging.Logger().Ctx(ctx).Warn().Print("git pull failed - console output was: ", r.sanitizedConsoleOutput())
+		r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git pull failed - returned error: %s", err.Error())
+
+		r.logContextErrorDetails(childCtxWithTimeout, "git pull", "childCtxWithTimeout")
+		r.logContextErrorDetails(ctx, "git pull", "ctx")
+
 		return err
 	}
 
@@ -327,13 +363,28 @@ func (r *Impl) Push(ctx context.Context) error {
 		return err
 	}
 
-	err = r.GitRepo.PushContext(childCtxWithTimeout, &git.PushOptions{
+	pushOpts := git.PushOptions{
 		Auth:       sshAuth,
 		Progress:   r, // implements io.Writer, sends to Debug logging
 		RemoteName: "origin",
-	})
+	}
+
+	if UseHTTP {
+		pushOpts.Auth = &http.BasicAuth{
+			Username: r.CustomConfiguration.BitbucketUsername(),
+			Password: r.CustomConfiguration.BitbucketPassword(),
+		}
+		pushOpts.InsecureSkipTLS = insecureSkipTLS
+	}
+
+	err = r.GitRepo.PushContext(childCtxWithTimeout, &pushOpts)
 	if err != nil && err != git.NoErrAlreadyUpToDate {
 		r.Logging.Logger().Ctx(ctx).Warn().Print("git push failed - console output was: ", r.sanitizedConsoleOutput())
+		r.Logging.Logger().Ctx(ctx).Warn().WithErr(err).Printf("git push failed - returned error: %s", err.Error())
+
+		r.logContextErrorDetails(childCtxWithTimeout, "git push", "childCtxWithTimeout")
+		r.logContextErrorDetails(ctx, "git push", "ctx")
+
 		return err
 	}
 
@@ -356,6 +407,15 @@ func (r *Impl) Discard(ctx context.Context) {
 	r.KnownCommits = make(map[string]bool)
 }
 
+func (r *Impl) logContextErrorDetails(ctx context.Context, operation string, contextName string) {
+	ctxCause := context.Cause(ctx)
+	if ctxCause != nil {
+		r.Logging.Logger().Ctx(ctx).Warn().WithErr(ctxCause).Printf("%s failed - %s cancellation cause: %s", operation, contextName, ctxCause.Error())
+	} else {
+		r.Logging.Logger().Ctx(ctx).Warn().Printf("%s failed - %s not cancelled", operation, contextName)
+	}
+}
+
 func (r *Impl) LastUpdated() time.Time {
 	r.mu.Lock()
 	defer r.mu.Unlock()

test/acceptance/configmock/configmock.go

@@ -78,6 +78,11 @@ func (c *MockConfig) SSHMetadataRepositoryUrl() string {
 	panic("implement me")
 }
 
+func (c *MockConfig) MetadataRepoUrl() string {
+	//TODO implement me
+	panic("implement me")
+}
+
 func (c *MockConfig) BitbucketUsername() string {
 	//TODO implement me
 	panic("implement me")

test/resources/valid-config-unique.yaml

@@ -24,6 +24,7 @@ AUTH_GROUP_WRITE: admin
 
 SSH_PRIVATE_KEY: some-private-key
 SSH_METADATA_REPO_URL: git://metadata
+METADATA_REPO_URL: http://metadata
 
 UPDATE_JOB_INTERVAL_MINUTES: 5
 UPDATE_JOB_TIMEOUT_SECONDS: 30

test/resources/valid-config.yaml

@@ -9,6 +9,7 @@ AUTH_OIDC_TOKEN_AUDIENCE: some-audience
 AUTH_GROUP_WRITE: admin
 
 SSH_METADATA_REPO_URL: git://metadata
+METADATA_REPO_URL: http://metadata
 
 UPDATE_JOB_INTERVAL_MINUTES: 5