Skip to content

Commit

Permalink
Merge remote-tracking branch 'harness0/master' into abhinav/add-expor…
Browse files Browse the repository at this point in the history
…ter (#7)
  • Loading branch information
abhinav-harness authored and Harness committed Jun 11, 2024
1 parent aa6950e commit 5087669
Show file tree
Hide file tree
Showing 19 changed files with 659 additions and 24 deletions.
3 changes: 3 additions & 0 deletions cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/harness/harness-migrate/cmd/cloudbuild"
"github.com/harness/harness-migrate/cmd/drone"
"github.com/harness/harness-migrate/cmd/github"
"github.com/harness/harness-migrate/cmd/gitimporter"
"github.com/harness/harness-migrate/cmd/gitlab"
"github.com/harness/harness-migrate/cmd/stash"
"github.com/harness/harness-migrate/cmd/terraform"
Expand Down Expand Up @@ -54,6 +55,8 @@ func Command() {
terraform.Register(app)
stash.Register(app)

gitimporter.Register(app)

app.Version(version)
kingpin.MustParse(app.Parse(os.Args[1:]))
}
159 changes: 159 additions & 0 deletions cmd/gitimporter/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitimporter

import (
"bufio"
"context"
"fmt"
"os"
"strings"

"github.com/harness/harness-migrate/cmd/util"
"github.com/harness/harness-migrate/internal/gitimporter"
"github.com/harness/harness-migrate/internal/tracer"
"github.com/harness/harness-migrate/types"

"github.com/alecthomas/kingpin/v2"
"github.com/google/uuid"
"golang.org/x/exp/slog"
)

type gitImport struct {
debug bool
trace bool

harnessEndpoint string
harnessToken string
harnessSpace string
//harnessRepo string

filePath string
}

type UserInvite bool

func (c *gitImport) run(*kingpin.ParseContext) error {
// create the logger
log := util.CreateLogger(c.debug)

// attach the logger to the context
ctx := context.Background()
ctx = slog.NewContext(ctx, log)

tracer_ := tracer.New()
defer tracer_.Close()

importUuid := uuid.New().String()
importer := gitimporter.NewImporter(c.harnessSpace, c.harnessToken, c.filePath, importUuid, tracer_)

tracer_.Log("starting operation with id: %s", importUuid)

repositoriesImportOutput, err := importer.UploadZip()

if err != nil {
tracer_.LogError("encountered error uploading zip: %s", err)
return err
}

if err := checkAndPerformUserInvite(repositoriesImportOutput, tracer_, importer); err != nil {
return err
}

if err := importer.IsComplete(); err != nil {
return err
}
return nil
}

func checkAndPerformUserInvite(repositoriesImportOutput *types.RepositoriesImportOutput, tracer_ tracer.Tracer, importer *gitimporter.Importer) error {
if repositoriesImportOutput != nil && len(repositoriesImportOutput.Users.NotPresent) != 0 {
tracer_.Log("Found users which are not in harness and are present in import data: ")
tracer_.Log("%v", repositoriesImportOutput.Users.NotPresent)
userFollowUp, err := doUserFollowUp()
if err != nil {
return err
}
if userFollowUp {
err = importer.InviteUsers(repositoriesImportOutput.Users.NotPresent)
if err != nil {
return err
}
}
}
return nil
}

func doUserFollowUp() (UserInvite, error) {
reader := bufio.NewReader(os.Stdin)

fmt.Println("Please select one of the following options:")
fmt.Println("1. Map missing user to yourself")
fmt.Println("2. Invite missing users (needs admin permission for space)")
fmt.Print("Enter your choice (1 or 2): ")

choice, _ := reader.ReadString('\n')
choice = strings.TrimSpace(choice)

switch choice {
case "1":
fmt.Println("You selected Option 1")
return false, nil
case "2":
fmt.Println("You selected Option 2")
return true, nil
default:
fmt.Println("Invalid choice. Please enter 1 or 2.")
return false, fmt.Errorf("invalid option selected for user invite")
}
}

func registerGitImporter(app *kingpin.CmdClause) {
c := new(gitImport)

cmd := app.Command("git-import", "import git data into harness/gitness").
Hidden().
Action(c.run)

cmd.Arg("filePath", "location of the zip file").
Required().
Envar("HARNESS_FILEPATH").
StringVar(&c.filePath)

cmd.Flag("harnessEndpoint", "url of harness code host").
Default("https://app.harness.io/").
Envar("HARNESS_HOST").
StringVar(&c.harnessEndpoint)

cmd.Flag("token", "harness api token").
Required().
Envar("HARNESS_TOKEN").
StringVar(&c.harnessToken)

cmd.Flag("space", "harness path where import should take place. Example: account/org/project").
Required().
Envar("HARNESS_TOKEN").
StringVar(&c.harnessSpace)

// cmd.Flag("repo", "Required in case of single repo import which already exists.").
// Envar("HARNESS_REPO").
// StringVar(&c.harnessRepo)

cmd.Flag("debug", "enable debug logging").
BoolVar(&c.debug)

cmd.Flag("trace", "enable trace logging").
BoolVar(&c.trace)
}
22 changes: 22 additions & 0 deletions cmd/gitimporter/importer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitimporter

import "github.com/alecthomas/kingpin/v2"

func Register(app *kingpin.Application) {
cmd := app.Command("git-import", "migrate data into harness from exported zip")
registerGitImporter(cmd)
}
6 changes: 3 additions & 3 deletions cmd/stash/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

"github.com/harness/harness-migrate/cmd/util"
"github.com/harness/harness-migrate/internal/checkpoint"
"github.com/harness/harness-migrate/internal/common"
"github.com/harness/harness-migrate/internal/gitexporter"
"github.com/harness/harness-migrate/internal/migrate/stash"
"github.com/harness/harness-migrate/internal/tracer"

Expand Down Expand Up @@ -88,7 +88,7 @@ func (c *exportCommand) run(*kingpin.ParseContext) error {
// extract the data
e := stash.New(client, c.stashOrg, checkpointManager, tracer_)

exporter := common.NewExporter(e, c.file)
exporter := gitexporter.NewExporter(e, c.file)
exporter.Export(ctx)
return nil
}
Expand All @@ -97,7 +97,7 @@ func (c *exportCommand) run(*kingpin.ParseContext) error {
func registerGit(app *kingpin.CmdClause) {
c := new(exportCommand)

cmd := app.Command("git", "export stash git data").
cmd := app.Command("git-export", "export stash git data").
Hidden().
Action(c.run)

Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ require (
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.4.0 // indirect
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gotidy/ptr v1.4.0 h1:7++suUs+HNHMnyz6/AW3SE+4EnBhupPSQTSI7QNijVc=
github.com/gotidy/ptr v1.4.0/go.mod h1:MjRBG6/IETiiZGWI8LrRtISXEji+8b/jigmj2q0mEyM=
github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package common
package gitexporter

import (
"github.com/harness/harness-migrate/internal/types"
Expand Down
46 changes: 31 additions & 15 deletions internal/common/exporter.go → internal/gitexporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package common
package gitexporter

import (
"context"
Expand All @@ -23,13 +23,16 @@ import (
"path/filepath"

"github.com/harness/harness-migrate/internal/codeerror"
"github.com/harness/harness-migrate/internal/common"
"github.com/harness/harness-migrate/internal/types"
"github.com/harness/harness-migrate/internal/util"

externalTypes "github.com/harness/harness-migrate/types"
)

const (
maxChunkSize = 25 * 1024 * 1024 // 25 MB
infoFileName = "info.json"
prFileName = "pr%d.json"
)

type Exporter struct {
Expand All @@ -46,19 +49,19 @@ func (e *Exporter) Export(ctx context.Context) {
path := filepath.Join(".", e.zipLocation)
err := util.CreateFolder(path)
if err != nil {
panic(fmt.Sprintf(PanicCannotCreateFolder, err))
panic(fmt.Sprintf(common.PanicCannotCreateFolder, err))
}
data, _ := e.getData(ctx)
for _, repo := range data {
err = e.writeJsonForRepo(repo)
err = e.writeJsonForRepo(mapRepoData(repo))
if err != nil {
panic(fmt.Sprintf(PanicWritingFileData, err))
panic(fmt.Sprintf(common.PanicWritingFileData, err))
}
}
}

// Calculate the size of the struct in bytes
func calculateSize(s *types.PullRequestData) int {
func calculateSize(s *externalTypes.PullRequestData) int {
data, err := json.Marshal(s)
// will never happen
if err != nil {
Expand All @@ -68,16 +71,16 @@ func calculateSize(s *types.PullRequestData) int {
}

// Split the array into smaller chunks if the size exceeds the maxChunkSize
func splitArray(arr []*types.PullRequestData) [][]*types.PullRequestData {
var chunks [][]*types.PullRequestData
var currentChunk []*types.PullRequestData
func splitArray(arr []*externalTypes.PullRequestData) [][]*externalTypes.PullRequestData {
var chunks [][]*externalTypes.PullRequestData
var currentChunk []*externalTypes.PullRequestData
currentSize := 0

for _, item := range arr {
itemSize := calculateSize(item)
if currentSize+itemSize > maxChunkSize {
chunks = append(chunks, currentChunk)
currentChunk = []*types.PullRequestData{}
currentChunk = []*externalTypes.PullRequestData{}
currentSize = 0
}
currentChunk = append(currentChunk, item)
Expand All @@ -89,15 +92,15 @@ func splitArray(arr []*types.PullRequestData) [][]*types.PullRequestData {
return chunks
}

func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error {
func (e *Exporter) writeJsonForRepo(repo *externalTypes.RepositoryData) error {
repoJson, _ := util.GetJson(repo.Repository)
pathRepo := filepath.Join(".", e.zipLocation, repo.Repository.RepoSlug)
pathRepo := filepath.Join(".", e.zipLocation, repo.Repository.Slug)
err := util.CreateFolder(pathRepo)
if err != nil {
return fmt.Errorf("cannot create folder")
}

err = util.WriteFile(filepath.Join(pathRepo, infoFileName), repoJson)
err = util.WriteFile(filepath.Join(pathRepo, externalTypes.InfoFileName), repoJson)
if err != nil {
return err
}
Expand All @@ -107,7 +110,7 @@ func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error {
}

prDataInSize := splitArray(repo.PullRequestData)
pathPR := filepath.Join(pathRepo, "pr")
pathPR := filepath.Join(pathRepo, externalTypes.PRFolderName)
err = util.CreateFolder(pathPR)
if err != nil {
return err
Expand All @@ -119,7 +122,7 @@ func (e *Exporter) writeJsonForRepo(repo *types.RepoData) error {
// todo: fix this
log.Printf("cannot serialize into json: %v", err)
}
prFilePath := fmt.Sprintf("pr%d.json", i)
prFilePath := fmt.Sprintf(prFileName, i)

err = util.WriteFile(filepath.Join(pathPR, prFilePath), prJson)
if err != nil {
Expand Down Expand Up @@ -174,3 +177,16 @@ func mapPrData(pr types.PRResponse, comments []types.PRComments) *types.PullRequ
Comments: comments,
}
}

func mapRepoData(repoData *types.RepoData) *externalTypes.RepositoryData {
d := new(externalTypes.RepositoryData)
d.Repository.Slug = repoData.Repository.RepoSlug
d.Repository.Repository = repoData.Repository.Repository

d.PullRequestData = make([]*externalTypes.PullRequestData, len(repoData.PullRequestData))
for i, prData := range repoData.PullRequestData {
d.PullRequestData[i].PullRequest.PullRequest = prData.PullRequest.PullRequest
// todo: map comment data
}
return d
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package common
package gitexporter

import (
"context"
Expand Down
Loading

0 comments on commit 5087669

Please sign in to comment.