fix: [CODE-2233]: graceful failure handling during import (#63)

internal/common/messages.go

@@ -47,6 +47,8 @@ const (
 	MsgCompleteImportPRs         = "Finished import %d pull requests with comments for repository %s."
 	MsgStartImportWebhooks       = "Starting importing webhooks for repository %s."
 	MsgCompleteImportWebhooks    = "Finished import %d webhooks for repository %s."
+	MsgStartRepoCleanup          = "Starting repo cleanup due to an incomplete export of %s"
+	MsgCompleteRepoCleanup       = "Finished repo cleanup due to an incomplete export of %s"
 
 	ErrGitClone                     = "cannot clone the git repository %q due to error: %w"
 	ErrGitFetch                     = "cannot fetch repository references for %s: %w"

internal/gitimporter/create_repo.go

@@ -11,7 +11,7 @@ import (
 )
 
 func (m *Importer) CreateRepo(
-	repo types.Repository,
+	repo *types.Repository,
 	targetSpace string,
 	tracer tracer.Tracer,
 ) (*harness.Repository, error) {

internal/gitimporter/importer.go

@@ -17,6 +17,7 @@ package gitimporter
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	filepath "path/filepath"
@@ -30,6 +31,8 @@ import (
 	"github.com/harness/harness-migrate/types"
 )
 
+var ErrAbortMigration = errors.New("aborting the migration. please checkout your command and try again")
+
 // Importer imports data from gitlab to Harness.
 type Importer struct {
 	Harness harness.Client
@@ -97,12 +100,27 @@ func (m *Importer) Import(ctx context.Context) error {
 	}
 
 	for _, f := range folders {
-		repo, err := m.createRepoAndDoPush(ctx, f)
+		repository, err := m.ReadRepoInfo(f)
 		if err != nil {
-			return fmt.Errorf("failed to create or push git data: %w", err)
+			m.Tracer.LogError("failed to read repo info from %q: %s", f, err.Error())
+			continue
+		}
+
+		repoRef := util.JoinPaths(m.HarnessSpace, repository.Name)
+
+		if err := m.createRepoAndDoPush(ctx, f, &repository); err != nil {
+			m.Tracer.LogError("failed to create or push git data for %q: %s", repoRef, err.Error())
+			if !errors.Is(err, harness.ErrDuplicate) {
+				// only cleanup if repo is not already existed (meaning was created by the migrator)
+				m.cleanup(repoRef, m.Tracer)
+			}
+			if notRecoverableError(err) {
+				return ErrAbortMigration
+			}
+
+			continue
 		}
 
-		repoRef := util.JoinPaths(m.HarnessSpace, repo.Name)
 		// update the repo state to migrate data import
 		_, err = m.Harness.UpdateRepositoryState(
 			repoRef,
@@ -112,9 +130,17 @@ func (m *Importer) Import(ctx context.Context) error {
 			return fmt.Errorf("failed to update the repo state to %s: %w", enum.RepoStateMigrateDataImport, err)
 		}
 
-		if !repo.IsEmpty {
-			if err := m.importRepoMetaData(ctx, repoRef, f); err != nil {
-				return fmt.Errorf("failed to import repo metadata: %w", err)
+		if !repository.IsEmpty {
+			err := m.importRepoMetaData(ctx, repoRef, f)
+			if err != nil {
+				m.Tracer.LogError("failed to import repo meta data for %q: %s", repoRef, err.Error())
+				// best effort delete the repo on server
+				m.cleanup(repoRef, m.Tracer)
+
+				if notRecoverableError(err) {
+					return ErrAbortMigration
+				}
+				continue
 			}
 		}
 
@@ -163,27 +189,22 @@ func (m *Importer) checkUsers(unzipLocation string) error {
 	return nil
 }
 
-func (m *Importer) createRepoAndDoPush(ctx context.Context, repoFolder string) (*types.Repository, error) {
-	repo, err := m.ReadRepoInfo(repoFolder)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read repo infos: %w", err)
-	}
-
+func (m *Importer) createRepoAndDoPush(ctx context.Context, repoFolder string, repo *types.Repository) error {
 	hRepo, err := m.CreateRepo(repo, m.HarnessSpace, m.Tracer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create repo %q: %w", repo.Slug, err)
+		return fmt.Errorf("failed to create repo %q: %w", repo.Slug, err)
 	}
 
 	if repo.IsEmpty {
-		return &repo, nil
+		return nil
 	}
 
 	err = m.Push(ctx, repoFolder, hRepo, m.Tracer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to push to repo: %w", err)
+		return fmt.Errorf("failed to push to repo: %w", err)
 	}
 
-	return &repo, nil
+	return nil
 }
 
 func (m *Importer) importRepoMetaData(_ context.Context, repoRef, repoFolder string) error {
@@ -202,6 +223,30 @@ func (m *Importer) importRepoMetaData(_ context.Context, repoRef, repoFolder str
 	return nil
 }
 
+// Cleanup cleans up the repo best effort.
+func (m *Importer) cleanup(repoRef string, tracer tracer.Tracer) {
+	tracer.Start(common.MsgStartRepoCleanup, repoRef)
+	err := m.Harness.DeleteRepository(repoRef)
+	if err != nil {
+		tracer.Stop("failed to clean up the repo on server: %w", err)
+		m.Tracer.LogError("failed to delete the repo %s: %s", repoRef, err.Error())
+	}
+
+	tracer.Stop(common.MsgCompleteRepoCleanup, repoRef)
+}
+
+// notRecoverableError checks if error is not recoverable, otherwise migration can continue
+func notRecoverableError(err error) bool {
+	if errors.Is(err, harness.ErrForbidden) ||
+		errors.Is(err, harness.ErrUnauthorized) ||
+		errors.Is(err, harness.ErrNotFound) ||
+		errors.Is(err, harness.ErrInvalidRef) {
+		return true
+	}
+
+	return false
+}
+
 func getRepoBaseFolders(directory string, singleRepo string) ([]string, error) {
 	var folders []string
 

internal/harness/client.go

@@ -76,11 +76,14 @@ type Client interface {
 	// CreateRepository creates a repository.
 	CreateRepository(parentRef string, repo *CreateRepositoryInput) (*Repository, error)
 
+	// DeleteRepository deletes a repository
+	DeleteRepository(repoRef string) error
+
 	// CreateRepositoryForMigration creates an empty repository ready for migration.
 	CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error)
 
 	// UpdateRepositoryState updates a repository state (for different steps of the migration).
-	UpdateRepositoryState(parentRef string, in *UpdateRepositoryStateInput) (*Repository, error)
+	UpdateRepositoryState(repoRef string, in *UpdateRepositoryStateInput) (*Repository, error)
 
 	// ImportPRs imports pull requests of a repository.
 	ImportPRs(repoRef string, in *types.PRsImportInput) error

internal/harness/client_impl.go

@@ -265,6 +265,25 @@ func (c *client) CreateRepository(parentRef string, repo *CreateRepositoryInput)
 	return out, nil
 }
 
+func (c *client) DeleteRepository(repoRef string) error {
+	queryParams, err := getQueryParamsFromRepoRef(path.Join(repoRef))
+	if err != nil {
+		return err
+	}
+
+	repoRef = strings.ReplaceAll(repoRef, pathSeparator, encodedPathSeparator)
+	uri := fmt.Sprintf("%s/api/v1/repos/%s?%s",
+		c.address,
+		repoRef,
+		queryParams,
+	)
+
+	if err := c.delete(uri); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *client) CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error) {
 	out := new(Repository)
 	queryParams, err := getQueryParamsFromRepoRef(path.Join(in.ParentRef, in.Identifier))

internal/harness/gitness_client.go

@@ -135,6 +135,19 @@ func (c *gitnessClient) CreateRepository(parentRef string, repo *CreateRepositor
 	return out, nil
 }
 
+func (c *gitnessClient) DeleteRepository(repoRef string) error {
+	repoRef = strings.ReplaceAll(repoRef, pathSeparator, encodedPathSeparator)
+	uri := fmt.Sprintf("%s/api/v1/repos/%s",
+		c.address,
+		repoRef,
+	)
+
+	if err := c.delete(uri); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *gitnessClient) CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error) {
 	out := new(Repository)
 	uri := fmt.Sprintf("%s/api/v1/migrate/repos",

internal/harness/http.go

@@ -17,6 +17,7 @@ package harness
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,6 +28,13 @@ import (
 	"strconv"
 )
 
+var (
+	ErrDuplicate    = errors.New("Resource already exists")
+	ErrNotFound     = errors.New("Resource not found")
+	ErrUnauthorized = errors.New("Unauthorized")
+	ErrForbidden    = errors.New("Forbidden")
+)
+
 // helper function to make an http request
 func Do(rawurl, method string, setAuth func(h *http.Header), in, out interface{}, tracing bool) error {
 	body, err := Open(rawurl, method, setAuth, in, out, tracing)
@@ -89,7 +97,20 @@ func Open(rawurl, method string, setAuth func(h *http.Header), in, out interface
 		os.Stdout.Write(dump)
 	}
 
-	if resp.StatusCode > 299 {
+	if resp.StatusCode < 299 {
+		return resp.Body, nil
+	}
+
+	switch resp.StatusCode {
+	case 401:
+		return nil, ErrUnauthorized
+	case 403:
+		return nil, ErrForbidden
+	case 404:
+		return nil, ErrNotFound
+	case 409:
+		return nil, ErrDuplicate
+	default:
 		defer resp.Body.Close()
 		out, _ := ioutil.ReadAll(resp.Body)
 		// attempt to unmarshal the error into the
@@ -101,5 +122,4 @@ func Open(rawurl, method string, setAuth func(h *http.Header), in, out interface
 		// else return the error body as a string
 		return nil, fmt.Errorf("client error %d: %s", resp.StatusCode, string(out))
 	}
-	return resp.Body, nil
 }

internal/harness/util.go

@@ -15,11 +15,14 @@
 package harness
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
 	"strings"
 )
 
+var ErrInvalidRef = errors.New("space reference is invalid")
+
 const (
 	pathSeparator        = "/"
 	encodedPathSeparator = "%252F"
@@ -36,7 +39,8 @@ func getQueryParamsFromRepoRef(repoRef string) (string, error) {
 	repoRefParts := strings.Split(s, "/")
 	// valid repoRef: "Acc/Repo", "Acc/Org/Repo", "Acc/Org/Projct/Repo"
 	if len(repoRefParts) < 2 || len(repoRefParts) > 4 {
-		return "", fmt.Errorf("repo ref %s segments is invalid, got %d want 2-4", repoRef, len(repoRefParts))
+		return "", fmt.Errorf("%w. reference %s has %d segments, want 2-4",
+			ErrInvalidRef, repoRef, len(repoRefParts))
 	}
 	params.Set(accountIdentifier, repoRefParts[0])
 	params.Set(routingId, repoRefParts[0])