Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[core] Skip writing empty deletion vector index #2982

Closed
wants to merge 1 commit into from
Closed

[core] Skip writing empty deletion vector index #2982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,10 @@ public Optional<DeletionVector> readDeletionVector(IndexFileMeta fileMeta, Strin
}

public IndexFileMeta writeDeletionVectorsIndex(Map<String, DeletionVector> deletionVectors) {
if (deletionVectors.isEmpty()) {
return IndexFileMeta.EMPTY_DV_INDEX;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

return null?

}

Pair<String, Map<String, Pair<Integer, Integer>>> pair =
deletionVectorsIndex.write(deletionVectors);
return new IndexFileMeta(
Expand Down
8 changes: 8 additions & 0 deletions paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,15 @@
import java.util.Map;
import java.util.Objects;

import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
import static org.apache.paimon.utils.SerializationUtils.newStringType;

/** Metadata of index file. */
public class IndexFileMeta {

public static final IndexFileMeta EMPTY_DV_INDEX =
new IndexFileMeta(DELETION_VECTORS_INDEX, "", 0, 0);

private final String indexType;
private final String fileName;
private final long fileSize;
Expand Down Expand Up @@ -83,6 +87,10 @@ public long rowCount() {
return deletionVectorsRanges;
}

public boolean isDeleted() {
return this.equals(EMPTY_DV_INDEX);
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down
2 changes: 1 addition & 1 deletion paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,7 @@ private void collectChanges(
f ->
appendIndexFiles.add(
new IndexManifestEntry(
FileKind.ADD,
f.isDeleted() ? FileKind.DELETE : FileKind.ADD,
commitMessage.partition(),
commitMessage.bucket(),
f)));
Expand Down
9 changes: 9 additions & 0 deletions ...k/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package org.apache.paimon.spark.sql

import org.apache.paimon.data.BinaryRow
import org.apache.paimon.deletionvectors.{DeletionVector, DeletionVectorsMaintainer}
import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX
import org.apache.paimon.manifest.IndexManifestEntry
import org.apache.paimon.spark.PaimonSparkTestBase

import org.apache.spark.sql.Row
Expand Down Expand Up @@ -60,6 +62,10 @@ class DeletionVectorTest extends PaimonSparkTestBase {
.deletionVectors()
}

def deletionVectorsIndexFiles(): java.util.List[IndexManifestEntry] = {
table.store().newIndexFileHandler().scan(DELETION_VECTORS_INDEX, BinaryRow.EMPTY_ROW)
}

val deletionVectors1 = restoreDeletionVectors()
// 1, 3 deleted, their row positions are 0, 2
Assertions.assertEquals(1, deletionVectors1.size())
Expand All @@ -70,13 +76,15 @@ class DeletionVectorTest extends PaimonSparkTestBase {
Assertions.assertTrue(e.getValue.isDeleted(0))
Assertions.assertTrue(e.getValue.isDeleted(2))
})
Assertions.assertEquals(1, deletionVectorsIndexFiles().size())

// Compact
// f3 (1, 2, 3), no deletion
spark.sql("CALL sys.compact('T')")
val deletionVectors2 = restoreDeletionVectors()
// After compaction, deletionVectors should be empty
Assertions.assertTrue(deletionVectors2.isEmpty)
Assertions.assertEquals(0, deletionVectorsIndexFiles().size())

// Insert2
// f3 (1, 2, 3), row with position 1 in f3 is marked deleted
Expand All @@ -95,6 +103,7 @@ class DeletionVectorTest extends PaimonSparkTestBase {
e => {
Assertions.assertTrue(e.getValue.isDeleted(1))
})
Assertions.assertEquals(1, deletionVectorsIndexFiles().size())
}
}

Expand Down
Loading