diff --git a/internal/common/messages.go b/internal/common/messages.go index f365796..2554aa6 100644 --- a/internal/common/messages.go +++ b/internal/common/messages.go @@ -47,6 +47,8 @@ const ( MsgCompleteImportPRs = "Finished import %d pull requests with comments for repository %s." MsgStartImportWebhooks = "Starting importing webhooks for repository %s." MsgCompleteImportWebhooks = "Finished import %d webhooks for repository %s." + MsgStartRepoCleanup = "Starting repo cleanup due to an incomplete export of %s" + MsgCompleteRepoCleanup = "Finished repo cleanup due to an incomplete export of %s" ErrGitClone = "cannot clone the git repository %q due to error: %w" ErrGitFetch = "cannot fetch repository references for %s: %w" diff --git a/internal/gitimporter/create_repo.go b/internal/gitimporter/create_repo.go index e023199..09cee4b 100644 --- a/internal/gitimporter/create_repo.go +++ b/internal/gitimporter/create_repo.go @@ -11,7 +11,7 @@ import ( ) func (m *Importer) CreateRepo( - repo types.Repository, + repo *types.Repository, targetSpace string, tracer tracer.Tracer, ) (*harness.Repository, error) { diff --git a/internal/gitimporter/importer.go b/internal/gitimporter/importer.go index 61f43b1..0cdf5cf 100644 --- a/internal/gitimporter/importer.go +++ b/internal/gitimporter/importer.go @@ -17,6 +17,7 @@ package gitimporter import ( "context" "encoding/json" + "errors" "fmt" "os" filepath "path/filepath" @@ -30,6 +31,8 @@ import ( "github.com/harness/harness-migrate/types" ) +var ErrAbortMigration = errors.New("aborting the migration. please checkout your command and try again") + // Importer imports data from gitlab to Harness. type Importer struct { Harness harness.Client @@ -97,12 +100,27 @@ func (m *Importer) Import(ctx context.Context) error { } for _, f := range folders { - repo, err := m.createRepoAndDoPush(ctx, f) + repository, err := m.ReadRepoInfo(f) if err != nil { - return fmt.Errorf("failed to create or push git data: %w", err) + m.Tracer.LogError("failed to read repo info from %q: %s", f, err.Error()) + continue + } + + repoRef := util.JoinPaths(m.HarnessSpace, repository.Name) + + if err := m.createRepoAndDoPush(ctx, f, &repository); err != nil { + m.Tracer.LogError("failed to create or push git data for %q: %s", repoRef, err.Error()) + if !errors.Is(err, harness.ErrDuplicate) { + // only cleanup if repo is not already existed (meaning was created by the migrator) + m.cleanup(repoRef, m.Tracer) + } + if notRecoverableError(err) { + return ErrAbortMigration + } + + continue } - repoRef := util.JoinPaths(m.HarnessSpace, repo.Name) // update the repo state to migrate data import _, err = m.Harness.UpdateRepositoryState( repoRef, @@ -112,9 +130,17 @@ func (m *Importer) Import(ctx context.Context) error { return fmt.Errorf("failed to update the repo state to %s: %w", enum.RepoStateMigrateDataImport, err) } - if !repo.IsEmpty { - if err := m.importRepoMetaData(ctx, repoRef, f); err != nil { - return fmt.Errorf("failed to import repo metadata: %w", err) + if !repository.IsEmpty { + err := m.importRepoMetaData(ctx, repoRef, f) + if err != nil { + m.Tracer.LogError("failed to import repo meta data for %q: %s", repoRef, err.Error()) + // best effort delete the repo on server + m.cleanup(repoRef, m.Tracer) + + if notRecoverableError(err) { + return ErrAbortMigration + } + continue } } @@ -163,27 +189,22 @@ func (m *Importer) checkUsers(unzipLocation string) error { return nil } -func (m *Importer) createRepoAndDoPush(ctx context.Context, repoFolder string) (*types.Repository, error) { - repo, err := m.ReadRepoInfo(repoFolder) - if err != nil { - return nil, fmt.Errorf("failed to read repo infos: %w", err) - } - +func (m *Importer) createRepoAndDoPush(ctx context.Context, repoFolder string, repo *types.Repository) error { hRepo, err := m.CreateRepo(repo, m.HarnessSpace, m.Tracer) if err != nil { - return nil, fmt.Errorf("failed to create repo %q: %w", repo.Slug, err) + return fmt.Errorf("failed to create repo %q: %w", repo.Slug, err) } if repo.IsEmpty { - return &repo, nil + return nil } err = m.Push(ctx, repoFolder, hRepo, m.Tracer) if err != nil { - return nil, fmt.Errorf("failed to push to repo: %w", err) + return fmt.Errorf("failed to push to repo: %w", err) } - return &repo, nil + return nil } func (m *Importer) importRepoMetaData(_ context.Context, repoRef, repoFolder string) error { @@ -202,6 +223,30 @@ func (m *Importer) importRepoMetaData(_ context.Context, repoRef, repoFolder str return nil } +// Cleanup cleans up the repo best effort. +func (m *Importer) cleanup(repoRef string, tracer tracer.Tracer) { + tracer.Start(common.MsgStartRepoCleanup, repoRef) + err := m.Harness.DeleteRepository(repoRef) + if err != nil { + tracer.Stop("failed to clean up the repo on server: %w", err) + m.Tracer.LogError("failed to delete the repo %s: %s", repoRef, err.Error()) + } + + tracer.Stop(common.MsgCompleteRepoCleanup, repoRef) +} + +// notRecoverableError checks if error is not recoverable, otherwise migration can continue +func notRecoverableError(err error) bool { + if errors.Is(err, harness.ErrForbidden) || + errors.Is(err, harness.ErrUnauthorized) || + errors.Is(err, harness.ErrNotFound) || + errors.Is(err, harness.ErrInvalidRef) { + return true + } + + return false +} + func getRepoBaseFolders(directory string, singleRepo string) ([]string, error) { var folders []string diff --git a/internal/harness/client.go b/internal/harness/client.go index 2084dab..4b2dfd3 100644 --- a/internal/harness/client.go +++ b/internal/harness/client.go @@ -76,11 +76,14 @@ type Client interface { // CreateRepository creates a repository. CreateRepository(parentRef string, repo *CreateRepositoryInput) (*Repository, error) + // DeleteRepository deletes a repository + DeleteRepository(repoRef string) error + // CreateRepositoryForMigration creates an empty repository ready for migration. CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error) // UpdateRepositoryState updates a repository state (for different steps of the migration). - UpdateRepositoryState(parentRef string, in *UpdateRepositoryStateInput) (*Repository, error) + UpdateRepositoryState(repoRef string, in *UpdateRepositoryStateInput) (*Repository, error) // ImportPRs imports pull requests of a repository. ImportPRs(repoRef string, in *types.PRsImportInput) error diff --git a/internal/harness/client_impl.go b/internal/harness/client_impl.go index ab4cf8e..f0d15e2 100644 --- a/internal/harness/client_impl.go +++ b/internal/harness/client_impl.go @@ -265,6 +265,25 @@ func (c *client) CreateRepository(parentRef string, repo *CreateRepositoryInput) return out, nil } +func (c *client) DeleteRepository(repoRef string) error { + queryParams, err := getQueryParamsFromRepoRef(path.Join(repoRef)) + if err != nil { + return err + } + + repoRef = strings.ReplaceAll(repoRef, pathSeparator, encodedPathSeparator) + uri := fmt.Sprintf("%s/api/v1/repos/%s?%s", + c.address, + repoRef, + queryParams, + ) + + if err := c.delete(uri); err != nil { + return err + } + return nil +} + func (c *client) CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error) { out := new(Repository) queryParams, err := getQueryParamsFromRepoRef(path.Join(in.ParentRef, in.Identifier)) diff --git a/internal/harness/gitness_client.go b/internal/harness/gitness_client.go index 66debee..1cd40ba 100644 --- a/internal/harness/gitness_client.go +++ b/internal/harness/gitness_client.go @@ -135,6 +135,19 @@ func (c *gitnessClient) CreateRepository(parentRef string, repo *CreateRepositor return out, nil } +func (c *gitnessClient) DeleteRepository(repoRef string) error { + repoRef = strings.ReplaceAll(repoRef, pathSeparator, encodedPathSeparator) + uri := fmt.Sprintf("%s/api/v1/repos/%s", + c.address, + repoRef, + ) + + if err := c.delete(uri); err != nil { + return err + } + return nil +} + func (c *gitnessClient) CreateRepositoryForMigration(in *CreateRepositoryForMigrateInput) (*Repository, error) { out := new(Repository) uri := fmt.Sprintf("%s/api/v1/migrate/repos", diff --git a/internal/harness/http.go b/internal/harness/http.go index c350f54..cd1ae81 100644 --- a/internal/harness/http.go +++ b/internal/harness/http.go @@ -17,6 +17,7 @@ package harness import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -27,6 +28,13 @@ import ( "strconv" ) +var ( + ErrDuplicate = errors.New("Resource already exists") + ErrNotFound = errors.New("Resource not found") + ErrUnauthorized = errors.New("Unauthorized") + ErrForbidden = errors.New("Forbidden") +) + // helper function to make an http request func Do(rawurl, method string, setAuth func(h *http.Header), in, out interface{}, tracing bool) error { body, err := Open(rawurl, method, setAuth, in, out, tracing) @@ -89,7 +97,20 @@ func Open(rawurl, method string, setAuth func(h *http.Header), in, out interface os.Stdout.Write(dump) } - if resp.StatusCode > 299 { + if resp.StatusCode < 299 { + return resp.Body, nil + } + + switch resp.StatusCode { + case 401: + return nil, ErrUnauthorized + case 403: + return nil, ErrForbidden + case 404: + return nil, ErrNotFound + case 409: + return nil, ErrDuplicate + default: defer resp.Body.Close() out, _ := ioutil.ReadAll(resp.Body) // attempt to unmarshal the error into the @@ -101,5 +122,4 @@ func Open(rawurl, method string, setAuth func(h *http.Header), in, out interface // else return the error body as a string return nil, fmt.Errorf("client error %d: %s", resp.StatusCode, string(out)) } - return resp.Body, nil } diff --git a/internal/harness/util.go b/internal/harness/util.go index 0f87555..5301880 100644 --- a/internal/harness/util.go +++ b/internal/harness/util.go @@ -15,11 +15,14 @@ package harness import ( + "errors" "fmt" "net/url" "strings" ) +var ErrInvalidRef = errors.New("space reference is invalid") + const ( pathSeparator = "/" encodedPathSeparator = "%252F" @@ -36,7 +39,8 @@ func getQueryParamsFromRepoRef(repoRef string) (string, error) { repoRefParts := strings.Split(s, "/") // valid repoRef: "Acc/Repo", "Acc/Org/Repo", "Acc/Org/Projct/Repo" if len(repoRefParts) < 2 || len(repoRefParts) > 4 { - return "", fmt.Errorf("repo ref %s segments is invalid, got %d want 2-4", repoRef, len(repoRefParts)) + return "", fmt.Errorf("%w. reference %s has %d segments, want 2-4", + ErrInvalidRef, repoRef, len(repoRefParts)) } params.Set(accountIdentifier, repoRefParts[0]) params.Set(routingId, repoRefParts[0])