fix lakectl client pagination (#614)

api/client.go

@@ -75,10 +75,10 @@ type RepositoryClient interface {
 	UploadObject(ctx context.Context, repository, branchID, path string, r io.Reader) (*models.ObjectStats, error)
 	DeleteObject(ctx context.Context, repository, branchID, path string) error
 
-	DiffRefs(ctx context.Context, repository, leftRef, rightRef string) ([]*models.Diff, error)
+	DiffRefs(ctx context.Context, repository, leftRef, rightRef string, after string, amount int) ([]*models.Diff, *models.Pagination, error)
 	Merge(ctx context.Context, repository, leftRef, rightRef string) (*models.MergeResult, error)
 
-	DiffBranch(ctx context.Context, repository, branch string) ([]*models.Diff, error)
+	DiffBranch(ctx context.Context, repository, branch string, after string, amount int) ([]*models.Diff, *models.Pagination, error)
 
 	GetRetentionPolicy(ctx context.Context, repository string) (*models.RetentionPolicyWithCreationDate, error)
 	UpdateRetentionPolicy(ctx context.Context, repository string, policy *models.RetentionPolicy) error
@@ -526,17 +526,20 @@ func (c *client) GetCommitLog(ctx context.Context, repository, branchID, after s
 	return resp.GetPayload().Results, resp.GetPayload().Pagination, nil
 }
 
-func (c *client) DiffRefs(ctx context.Context, repository, leftRef, rightRef string) ([]*models.Diff, error) {
+func (c *client) DiffRefs(ctx context.Context, repository, leftRef, rightRef, after string, amount int) ([]*models.Diff, *models.Pagination, error) {
 	diff, err := c.remote.Refs.DiffRefs(&refs.DiffRefsParams{
+		After:      swag.String(after),
+		Amount:     swag.Int64(int64(amount)),
 		LeftRef:    leftRef,
-		RightRef:   rightRef,
 		Repository: repository,
+		RightRef:   rightRef,
 		Context:    ctx,
 	}, c.auth)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return diff.GetPayload().Results, nil
+	payload := diff.GetPayload()
+	return payload.Results, payload.Pagination, nil
 }
 
 func (c *client) Merge(ctx context.Context, repository, leftRef, rightRef string) (*models.MergeResult, error) {
@@ -557,17 +560,21 @@ func (c *client) Merge(ctx context.Context, repository, leftRef, rightRef string
 	return nil, err
 }
 
-func (c *client) DiffBranch(ctx context.Context, repoID, branch string) ([]*models.Diff, error) {
+func (c *client) DiffBranch(ctx context.Context, repoID, branch string, after string, amount int) ([]*models.Diff, *models.Pagination, error) {
 	diff, err := c.remote.Branches.DiffBranch(&branches.DiffBranchParams{
+		After:      swag.String(after),
+		Amount:     swag.Int64(int64(amount)),
 		Branch:     branch,
 		Repository: repoID,
 		Context:    ctx,
 	}, c.auth)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return diff.GetPayload().Results, nil
+	payload := diff.GetPayload()
+	return payload.Results, payload.Pagination, nil
 }
+
 func (c *client) Symlink(ctx context.Context, repoID, branch, path string) (string, error) {
 	resp, err := c.remote.Metadata.CreateSymlink(&metadata.CreateSymlinkParams{
 		Location:   swag.String(path),
@@ -580,6 +587,7 @@ func (c *client) Symlink(ctx context.Context, repoID, branch, path string) (stri
 	}
 	return resp.GetPayload(), nil
 }
+
 func (c *client) GetRetentionPolicy(ctx context.Context, repository string) (*models.RetentionPolicyWithCreationDate, error) {
 	policy, err := c.remote.Retention.GetRetentionPolicy(&retention.GetRetentionPolicyParams{
 		Repository: repository,

cmd/lakectl/cmd/diff.go

@@ -4,15 +4,18 @@ import (
 	"context"
 	"os"
 
+	"github.com/go-openapi/swag"
 	"github.com/jedib0t/go-pretty/text"
 	"github.com/spf13/cobra"
+	"github.com/treeverse/lakefs/api"
 	"github.com/treeverse/lakefs/api/gen/models"
 	"github.com/treeverse/lakefs/uri"
 )
 
 const (
 	diffCmdMinArgs = 1
 	diffCmdMaxArgs = 2
+	diffPageSize   = 100
 )
 
 var diffCmd = &cobra.Command{
@@ -26,40 +29,59 @@ var diffCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		client := getClient()
 
-		var diff []*models.Diff
-		var err error
 		const diffWithOtherArgsCount = 2
 		if len(args) == diffWithOtherArgsCount {
 			if err := IsRefURI(1)(args); err != nil {
 				DieErr(err)
 			}
 			leftRefURI := uri.Must(uri.Parse(args[0]))
 			rightRefURI := uri.Must(uri.Parse(args[1]))
-
 			if leftRefURI.Repository != rightRefURI.Repository {
 				Die("both references must belong to the same repository", 1)
 			}
-
-			diff, err = client.DiffRefs(context.Background(), leftRefURI.Repository, leftRefURI.Ref, rightRefURI.Ref)
-			if err != nil {
-				DieErr(err)
-			}
-			for _, line := range diff {
-				FmtDiff(line, true)
-			}
+			printDiffRefs(client, leftRefURI.Repository, leftRefURI.Ref, rightRefURI.Ref)
 		} else {
 			branchURI := uri.Must(uri.Parse(args[0]))
-			diff, err = client.DiffBranch(context.Background(), branchURI.Repository, branchURI.Ref)
-			if err != nil {
-				DieErr(err)
-			}
-			for _, line := range diff {
-				FmtDiff(line, false)
-			}
+			printDiffBranch(client, branchURI.Repository, branchURI.Ref)
 		}
 	},
 }
 
+func printDiffBranch(client api.Client, repository string, branch string) {
+	var after string
+	for {
+		diff, pagination, err := client.DiffBranch(context.Background(), repository, branch, after, diffPageSize)
+		if err != nil {
+			DieErr(err)
+		}
+		for _, line := range diff {
+			FmtDiff(line, false)
+		}
+		if !swag.BoolValue(pagination.HasMore) {
+			break
+		}
+		after = pagination.NextOffset
+	}
+}
+
+func printDiffRefs(client api.Client, repository string, leftRef string, rightRef string) {
+	var after string
+	for {
+		diff, pagination, err := client.DiffRefs(context.Background(), repository, leftRef, rightRef,
+			after, diffPageSize)
+		if err != nil {
+			DieErr(err)
+		}
+		for _, line := range diff {
+			FmtDiff(line, true)
+		}
+		if !swag.BoolValue(pagination.HasMore) {
+			break
+		}
+		after = pagination.NextOffset
+	}
+}
+
 func FmtDiff(diff *models.Diff, withDirection bool) {
 	var color text.Color
 	var action string