diff --git a/pkg/meta/backup.go b/pkg/meta/backup.go index 0893518afb3a..82cf01eec0e4 100644 --- a/pkg/meta/backup.go +++ b/pkg/meta/backup.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io" + "reflect" "sync" "unsafe" @@ -129,11 +130,12 @@ func (f *BakFormat) WriteSegment(w io.Writer, seg *BakSegment) error { name := seg.String() info, ok := f.Footer.Msg.Infos[name] if !ok { - info = &pb.Footer_SegInfo{Offset: []uint64{}} + info = &pb.Footer_SegInfo{Offset: []uint64{}, Num: 0} f.Footer.Msg.Infos[name] = info } info.Offset = append(info.Offset, f.Offset) + info.Num += seg.Num() f.Offset += uint64(n) return nil } @@ -177,6 +179,7 @@ func (f *BakFormat) ReadFooter(r io.ReadSeeker) (*BakFooter, error) { if footer.Msg.Magic != BakMagic { return nil, fmt.Errorf("invalid magic number %d, expect %d", footer.Msg.Magic, BakMagic) } + f.Footer = footer return footer, nil } @@ -198,7 +201,7 @@ func (h *BakFooter) Marshal() ([]byte, error) { func (h *BakFooter) Unmarshal(r io.ReadSeeker) error { lenSize := int64(unsafe.Sizeof(h.Len)) - _, _ = r.Seek(lenSize, io.SeekEnd) + _, _ = r.Seek(-lenSize, io.SeekEnd) data := make([]byte, lenSize) if n, err := r.Read(data); err != nil && n != int(lenSize) { @@ -206,12 +209,13 @@ func (h *BakFooter) Unmarshal(r io.ReadSeeker) error { } h.Len = binary.BigEndian.Uint64(data) - _, _ = r.Seek(int64(h.Len)+lenSize, io.SeekEnd) + _, _ = r.Seek(-int64(h.Len)-lenSize, io.SeekEnd) data = make([]byte, h.Len) if n, err := r.Read(data); err != nil && n != int(h.Len) { return fmt.Errorf("failed to read footer: err %w, read len %d, expect len %d", err, n, h.Len) } + h.Msg = &pb.Footer{} if err := proto.Unmarshal(data, h.Msg); err != nil { return fmt.Errorf("failed to unmarshal footer: %w", err) } @@ -228,6 +232,25 @@ func (s *BakSegment) String() string { return string(proto.MessageName(s.Val).Name()) } +func (s *BakSegment) Num() uint64 { + switch v := s.Val.(type) { + case *pb.Format: + return 1 + case *pb.Counters: + return 6 + default: + val := reflect.ValueOf(v) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + field := val.FieldByName("List") + if field.IsValid() && field.Kind() == reflect.Slice { + return uint64(field.Len()) + } + return 0 + } +} + func (s *BakSegment) Marshal(w io.Writer) (int, error) { if s == nil || s.Val == nil { return 0, fmt.Errorf("segment %s is nil", s) @@ -304,7 +327,7 @@ func (opt *DumpOption) check() *DumpOption { opt = &DumpOption{} } if opt.CoNum < 1 { - opt.CoNum = 1 + opt.CoNum = 10 } return opt } @@ -366,7 +389,7 @@ type LoadOption struct { func (opt *LoadOption) check() { if opt.CoNum < 1 { - opt.CoNum = 1 + opt.CoNum = 10 } } @@ -406,7 +429,6 @@ type txMaxRetryKey struct{} type bTxnOption struct { coNum int notUsed bool - readOnly bool maxRetry int maxStmtRetry int } diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 20060c085ecb..f492a7d5c2ae 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -3097,7 +3097,6 @@ func (m *baseMeta) DumpMetaV2(ctx Context, w io.Writer, opt *DumpOption) error { en: m.en, opt: &bTxnOption{ coNum: opt.CoNum, - readOnly: true, maxRetry: 1, maxStmtRetry: 3, }, @@ -3185,7 +3184,7 @@ func (m *baseMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error { go workerFunc(ctx, taskCh) } - bak := NewBakFormat() + bak := &BakFormat{} for { seg, err := bak.ReadSegment(r) if err != nil { diff --git a/pkg/meta/pb/backup.pb.go b/pkg/meta/pb/backup.pb.go index be94006a80ef..66cb8daecb24 100644 --- a/pkg/meta/pb/backup.pb.go +++ b/pkg/meta/pb/backup.pb.go @@ -1490,6 +1490,7 @@ type Footer_SegInfo struct { unknownFields protoimpl.UnknownFields Offset []uint64 `protobuf:"varint,1,rep,packed,name=offset,proto3" json:"offset,omitempty"` + Num uint64 `protobuf:"varint,2,opt,name=num,proto3" json:"num,omitempty"` } func (x *Footer_SegInfo) Reset() { @@ -1529,6 +1530,13 @@ func (x *Footer_SegInfo) GetOffset() []uint64 { return nil } +func (x *Footer_SegInfo) GetNum() uint64 { + if x != nil { + return x.Num + } + return 0 +} + var File_pkg_meta_pb_backup_proto protoreflect.FileDescriptor var file_pkg_meta_pb_backup_proto_rawDesc = []byte{ @@ -1641,21 +1649,22 @@ var file_pkg_meta_pb_backup_proto_rawDesc = []byte{ 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x2e, 0x0a, 0x0b, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x52, - 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xd6, 0x01, 0x0a, 0x06, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, + 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xe8, 0x01, 0x0a, 0x06, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x2e, 0x49, 0x6e, 0x66, 0x6f, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x21, 0x0a, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x1a, 0x33, 0x0a, 0x07, 0x53, 0x65, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x1a, 0x4c, 0x0a, 0x0a, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x67, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x06, - 0x5a, 0x04, 0x2e, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x10, 0x0a, 0x03, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6e, + 0x75, 0x6d, 0x1a, 0x4c, 0x0a, 0x0a, 0x49, 0x6e, 0x66, 0x6f, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x70, 0x62, 0x2e, 0x46, 0x6f, 0x6f, 0x74, 0x65, 0x72, 0x2e, 0x53, 0x65, + 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/meta/pb/backup.proto b/pkg/meta/pb/backup.proto index d26d53c4917d..a9d365164c6f 100644 --- a/pkg/meta/pb/backup.proto +++ b/pkg/meta/pb/backup.proto @@ -145,6 +145,7 @@ message SymlinkList { message Footer { message SegInfo { repeated uint64 offset = 1; + uint64 num = 2; } uint32 magic = 1; diff --git a/pkg/meta/sql_bak.go b/pkg/meta/sql_bak.go index c887694aabc7..0bdde23202a7 100644 --- a/pkg/meta/sql_bak.go +++ b/pkg/meta/sql_bak.go @@ -35,7 +35,7 @@ import ( ) var ( - sqlDumpBatchSize = 40960 + sqlDumpBatchSize = 100000 ) func (m *dbMeta) buildDumpedSeg(typ int, opt *DumpOption, txn *eTxn) iDumpedSeg { @@ -89,35 +89,36 @@ func (m *dbMeta) buildLoadedPools(typ int) []*sync.Pool { } func (m *dbMeta) buildLoadedSeg(typ int, opt *LoadOption) iLoadedSeg { + ls := loadedSeg{typ: typ, meta: m} switch typ { case SegTypeFormat: - return &sqlFormatLS{loadedSeg{typ: typ, meta: m}} + return &sqlFormatLS{ls} case SegTypeCounter: - return &sqlCounterLS{loadedSeg{typ: typ, meta: m}} + return &sqlCounterLS{ls} case SegTypeSustained: - return &sqlSustainedLS{loadedSeg{typ: typ, meta: m}} + return &sqlSustainedLS{ls} case SegTypeDelFile: - return &sqlDelFileLS{loadedSeg{typ: typ, meta: m}} + return &sqlDelFileLS{ls} case SegTypeSliceRef: - return &sqlSliceRefLS{loadedSeg{typ: typ, meta: m}} + return &sqlSliceRefLS{ls} case SegTypeAcl: - return &sqlAclLS{loadedSeg{typ: typ, meta: m}} + return &sqlAclLS{ls} case SegTypeXattr: - return &sqlXattrLS{loadedSeg{typ: typ, meta: m}} + return &sqlXattrLS{ls} case SegTypeQuota: - return &sqlQuotaLS{loadedSeg{typ: typ, meta: m}} + return &sqlQuotaLS{ls} case SegTypeStat: - return &sqlStatLS{loadedSeg{typ: typ, meta: m}} + return &sqlStatLS{ls} case SegTypeNode: - return &sqlNodeLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)} + return &sqlNodeLS{ls, m.buildLoadedPools(typ)} case SegTypeChunk: - return &sqlChunkLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)} + return &sqlChunkLS{ls, m.buildLoadedPools(typ)} case SegTypeEdge: - return &sqlEdgeLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)} + return &sqlEdgeLS{ls, m.buildLoadedPools(typ)} case SegTypeParent: - return &sqlParentLS{loadedSeg{typ: typ, meta: m}} + return &sqlParentLS{ls} case SegTypeSymlink: - return &sqlSymlinkLS{loadedSeg{typ: typ, meta: m}, m.buildLoadedPools(typ)} + return &sqlSymlinkLS{ls, m.buildLoadedPools(typ)} } return nil }