meta: fix protection check for compaction (#4349)

juicedata · Jan 16, 2024 · 3782c07 · 3782c07
1 parent 1616431
commit 3782c07
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 15 deletions.
diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go
@@ -1239,6 +1239,53 @@ func testCompaction(t *testing.T, m Meta, trash bool) {
 	if deletes < 200 {
 		t.Fatalf("deleted slices %d is less than 200", deletes)
 	}
+
+	// truncate to 0
+	if st := m.Truncate(ctx, inode, 0, 0, attr, false); st != 0 {
+		t.Fatalf("truncate file: %s", st)
+	}
+	if c, ok := m.(compactor); ok {
+		c.compactChunk(inode, 0, true)
+	}
+	if st := m.Read(ctx, inode, 0, &slices); st != 0 {
+		t.Fatalf("read 0: %s", st)
+	}
+	if len(slices) != 1 || slices[0].Len != 1 {
+		t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len)
+	}
+
+	if st := m.Truncate(ctx, inode, 0, 64<<10, attr, false); st != 0 {
+		t.Fatalf("truncate file: %s", st)
+	}
+	m.NewSlice(ctx, &sliceId)
+	_ = m.Write(ctx, inode, 0, uint32(1<<20), Slice{Id: sliceId, Size: 2 << 20, Len: 2 << 20}, time.Now())
+	if c, ok := m.(compactor); ok {
+		c.compactChunk(inode, 0, true)
+	}
+	if st := m.Read(ctx, inode, 0, &slices); st != 0 {
+		t.Fatalf("read 0: %s", st)
+	}
+	if len(slices) != 2 || slices[0].Id != 0 || slices[1].Len != 2<<20 {
+		t.Fatalf("inode %d should be compacted, but have %d slices, id %d size %d",
+			inode, len(slices), slices[0].Id, slices[1].Len)
+	}
+
+	m.NewSlice(ctx, &sliceId)
+	_ = m.Write(ctx, inode, 0, uint32(512<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 64 << 10}, time.Now())
+	m.NewSlice(ctx, &sliceId)
+	_ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}, time.Now())
+	m.NewSlice(ctx, &sliceId)
+	_ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}, time.Now())
+	_ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}, time.Now())
+	if c, ok := m.(compactor); ok {
+		c.compactChunk(inode, 0, true)
+	}
+	if st := m.Read(ctx, inode, 0, &slices); st != 0 {
+		t.Fatalf("read 0: %s", st)
+	}
+	if len(slices) != 1 || slices[0].Len != 3<<20 {
+		t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len)
+	}
 }
 
 func testConcurrentWrite(t *testing.T, m Meta) {

diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go
@@ -2923,17 +2923,17 @@ func (m *redisMeta) compactChunk(inode Ino, indx uint32, force bool) {
 		return
 	}
 	skipped := skipSome(ss)
-	var last *slice
+	var first, last *slice
 	if skipped > 0 {
-		last = ss[skipped-1]
+		first, last = ss[0], ss[skipped-1]
 	}
 	ss = ss[skipped:]
 	pos, size, slices := compactChunk(ss)
 	if len(ss) < 2 || size == 0 {
 		return
 	}
-	if last != nil && last.pos+last.len > pos {
-		panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos))
+	if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos {
+		panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size))
 	}
 
 	var id uint64

diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go
@@ -157,11 +157,12 @@ func buildSlice(ss []*slice) []Slice {
 func compactChunk(ss []*slice) (uint32, uint32, []Slice) {
 	var chunk = buildSlice(ss)
 	var pos uint32
+	for len(chunk) > 1 && chunk[0].Id == 0 {
+		pos += chunk[0].Len
+		chunk = chunk[1:]
+	}
 	if len(chunk) == 1 && chunk[0].Id == 0 {
 		chunk[0].Len = 1
-	} else if len(chunk) > 1 && chunk[0].Id == 0 {
-		pos = chunk[0].Len
-		chunk = chunk[1:]
 	}
 	var size uint32
 	for _, c := range chunk {

diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
@@ -2783,17 +2783,17 @@ func (m *dbMeta) compactChunk(inode Ino, indx uint32, force bool) {
 		return
 	}
 	skipped := skipSome(ss)
-	var last *slice
+	var first, last *slice
 	if skipped > 0 {
-		last = ss[skipped-1]
+		first, last = ss[0], ss[skipped-1]
 	}
 	ss = ss[skipped:]
 	pos, size, slices := compactChunk(ss)
 	if len(ss) < 2 || size == 0 {
 		return
 	}
-	if last != nil && last.pos+last.len > pos {
-		panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos))
+	if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos {
+		panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size))
 	}
 
 	var id uint64

diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go
@@ -2410,17 +2410,17 @@ func (m *kvMeta) compactChunk(inode Ino, indx uint32, force bool) {
 		return
 	}
 	skipped := skipSome(ss)
-	var last *slice
+	var first, last *slice
 	if skipped > 0 {
-		last = ss[skipped-1]
+		first, last = ss[0], ss[skipped-1]
 	}
 	ss = ss[skipped:]
 	pos, size, slices := compactChunk(ss)
 	if len(ss) < 2 || size == 0 {
 		return
 	}
-	if last != nil && last.pos+last.len > pos {
-		panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos))
+	if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos {
+		panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size))
 	}
 
 	var id uint64