meta/sql: optimise backup config and format (#5348)

Signed-off-by: jiefenghuang <[email protected]>

pkg/meta/backup.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"reflect"
 	"sync"
 	"unsafe"
 
@@ -129,11 +130,12 @@ func (f *BakFormat) WriteSegment(w io.Writer, seg *BakSegment) error {
 	name := seg.String()
 	info, ok := f.Footer.Msg.Infos[name]
 	if !ok {
-		info = &pb.Footer_SegInfo{Offset: []uint64{}}
+		info = &pb.Footer_SegInfo{Offset: []uint64{}, Num: 0}
 		f.Footer.Msg.Infos[name] = info
 	}
 
 	info.Offset = append(info.Offset, f.Offset)
+	info.Num += seg.Num()
 	f.Offset += uint64(n)
 	return nil
 }
@@ -177,6 +179,7 @@ func (f *BakFormat) ReadFooter(r io.ReadSeeker) (*BakFooter, error) {
 	if footer.Msg.Magic != BakMagic {
 		return nil, fmt.Errorf("invalid magic number %d, expect %d", footer.Msg.Magic, BakMagic)
 	}
+	f.Footer = footer
 	return footer, nil
 }
 
@@ -198,20 +201,21 @@ func (h *BakFooter) Marshal() ([]byte, error) {
 
 func (h *BakFooter) Unmarshal(r io.ReadSeeker) error {
 	lenSize := int64(unsafe.Sizeof(h.Len))
-	_, _ = r.Seek(lenSize, io.SeekEnd)
+	_, _ = r.Seek(-lenSize, io.SeekEnd)
 
 	data := make([]byte, lenSize)
 	if n, err := r.Read(data); err != nil && n != int(lenSize) {
 		return fmt.Errorf("failed to read footer length: err %w, read len %d, expect len %d", err, n, lenSize)
 	}
 
 	h.Len = binary.BigEndian.Uint64(data)
-	_, _ = r.Seek(int64(h.Len)+lenSize, io.SeekEnd)
+	_, _ = r.Seek(-int64(h.Len)-lenSize, io.SeekEnd)
 	data = make([]byte, h.Len)
 	if n, err := r.Read(data); err != nil && n != int(h.Len) {
 		return fmt.Errorf("failed to read footer: err %w, read len %d, expect len %d", err, n, h.Len)
 	}
 
+	h.Msg = &pb.Footer{}
 	if err := proto.Unmarshal(data, h.Msg); err != nil {
 		return fmt.Errorf("failed to unmarshal footer: %w", err)
 	}
@@ -228,6 +232,25 @@ func (s *BakSegment) String() string {
 	return string(proto.MessageName(s.Val).Name())
 }
 
+func (s *BakSegment) Num() uint64 {
+	switch v := s.Val.(type) {
+	case *pb.Format:
+		return 1
+	case *pb.Counters:
+		return 6
+	default:
+		val := reflect.ValueOf(v)
+		if val.Kind() == reflect.Ptr {
+			val = val.Elem()
+		}
+		field := val.FieldByName("List")
+		if field.IsValid() && field.Kind() == reflect.Slice {
+			return uint64(field.Len())
+		}
+		return 0
+	}
+}
+
 func (s *BakSegment) Marshal(w io.Writer) (int, error) {
 	if s == nil || s.Val == nil {
 		return 0, fmt.Errorf("segment %s is nil", s)
@@ -304,7 +327,7 @@ func (opt *DumpOption) check() *DumpOption {
 		opt = &DumpOption{}
 	}
 	if opt.CoNum < 1 {
-		opt.CoNum = 1
+		opt.CoNum = 10
 	}
 	return opt
 }
@@ -366,7 +389,7 @@ type LoadOption struct {
 
 func (opt *LoadOption) check() {
 	if opt.CoNum < 1 {
-		opt.CoNum = 1
+		opt.CoNum = 10
 	}
 }
 
@@ -406,7 +429,6 @@ type txMaxRetryKey struct{}
 type bTxnOption struct {
 	coNum        int
 	notUsed      bool
-	readOnly     bool
 	maxRetry     int
 	maxStmtRetry int
 }

pkg/meta/base.go

@@ -3097,7 +3097,6 @@ func (m *baseMeta) DumpMetaV2(ctx Context, w io.Writer, opt *DumpOption) error {
 			en: m.en,
 			opt: &bTxnOption{
 				coNum:        opt.CoNum,
-				readOnly:     true,
 				maxRetry:     1,
 				maxStmtRetry: 3,
 			},
@@ -3185,7 +3184,7 @@ func (m *baseMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error {
 		go workerFunc(ctx, taskCh)
 	}
 
-	bak := NewBakFormat()
+	bak := &BakFormat{}
 	for {
 		seg, err := bak.ReadSegment(r)
 		if err != nil {

pkg/meta/pb/backup.proto

@@ -145,6 +145,7 @@ message SymlinkList {
 message Footer {
   message SegInfo {
     repeated uint64 offset = 1;
+    uint64 num = 2;
   }
 
   uint32 magic = 1;

pkg/meta/sql_bak.go

@@ -35,7 +35,7 @@ import (
 )
 
 var (
-	sqlDumpBatchSize = 40960
+	sqlDumpBatchSize = 100000
 )
 
 func (m *dbMeta) buildDumpedSeg(typ int, opt *DumpOption, txn *eTxn) iDumpedSeg {
@@ -89,35 +89,36 @@ func (m *dbMeta) buildLoadedPools(typ int) []*sync.Pool {
 }
 
 func (m *dbMeta) buildLoadedSeg(typ int, opt *LoadOption) iLoadedSeg {
+	ls := loadedSeg{typ: typ, meta: m}
 	switch typ {
 	case SegTypeFormat:
-		return &sqlFormatLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlFormatLS{ls}
 	case SegTypeCounter:
-		return &sqlCounterLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlCounterLS{ls}
 	case SegTypeSustained:
-		return &sqlSustainedLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlSustainedLS{ls}
 	case SegTypeDelFile:
-		return &sqlDelFileLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlDelFileLS{ls}
 	case SegTypeSliceRef:
-		return &sqlSliceRefLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlSliceRefLS{ls}
 	case SegTypeAcl:
-		return &sqlAclLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlAclLS{ls}
 	case SegTypeXattr:
-		return &sqlXattrLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlXattrLS{ls}
 	case SegTypeQuota:
-		return &sqlQuotaLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlQuotaLS{ls}
 	case SegTypeStat:
-		return &sqlStatLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlStatLS{ls}
 	case SegTypeNode:
-		return &sqlNodeLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)}
+		return &sqlNodeLS{ls, m.buildLoadedPools(typ)}
 	case SegTypeChunk:
-		return &sqlChunkLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)}
+		return &sqlChunkLS{ls, m.buildLoadedPools(typ)}
 	case SegTypeEdge:
-		return &sqlEdgeLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)}
+		return &sqlEdgeLS{ls, m.buildLoadedPools(typ)}
 	case SegTypeParent:
-		return &sqlParentLS{loadedSeg{typ: typ, meta: m}}
+		return &sqlParentLS{ls}
 	case SegTypeSymlink:
-		return &sqlSymlinkLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)}
+		return &sqlSymlinkLS{ls, m.buildLoadedPools(typ)}
 	}
 	return nil
 }