Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zouxxyy
Zouxxyy committed Feb 1, 2024
1 parent 4b667aa commit 12b3d73
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 11 deletions.
12 changes: 6 additions & 6 deletions docs/content/how-to/system-tables.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ You can query the snapshot history information of the table through snapshots ta
SELECT * FROM MyTable$snapshots;

/*
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
| snapshot_id | schema_id | commit_user | commit_identifier | commit_kind | commit_time | base_manifest_list | delta_manifest_list | changelog_manifest_list | total_record_count | delta_record_count | changelog_record_count | added_file_count | delete_file_count | watermark |
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
| 2 | 0 | 7ca4cd28-98e... | 2 | APPEND | 2022-10-26 11:44:15.600 | manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... | 2 | 2 | 0 | 2 | 0 | 1666755855600 |
| 1 | 0 | 870062aa-3e9... | 1 | APPEND | 2022-10-26 11:44:15.148 | manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... | 1 | 1 | 0 | 1 | 0 | 1666755855148 |
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+--------------------+
| snapshot_id | schema_id | commit_user | commit_identifier | commit_kind | commit_time | base_manifest_list | delta_manifest_list | changelog_manifest_list | total_record_count | delta_record_count | changelog_record_count | added_file_count | delete_file_count | watermark | statistics |
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+--------------------+
| 2 | 0 | 7ca4cd28-98e... | 2 | APPEND | 2022-10-26 11:44:15.600 | manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... | manifest-list-31323d5f-76e6... | 2 | 2 | 0 | 2 | 0 | 1666755855600 |{\n "snapshotId"...|
| 1 | 0 | 870062aa-3e9... | 1 | APPEND | 2022-10-26 11:44:15.148 | manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... | manifest-list-31593d5f-76e6... | 1 | 1 | 0 | 1 | 0 | 1666755855148 || NULL|
+--------------+------------+-----------------+-------------------+--------------+-------------------------+--------------------------------+------------------------------- +--------------------------------+---------------------+---------------------+-------------------------+-------------------+--------------------+----------------+--------------------+
2 rows in set
*/
```
Expand Down
20 changes: 16 additions & 4 deletions paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.stats.StatsFileHandler;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.ReadonlyTable;
import org.apache.paimon.table.Table;
Expand Down Expand Up @@ -98,7 +100,9 @@ public class SnapshotsTable implements ReadonlyTable {
new DataField(11, "changelog_record_count", new BigIntType(true)),
new DataField(12, "added_file_count", new IntType(true)),
new DataField(13, "delete_file_count", new IntType(true)),
new DataField(14, "watermark", new BigIntType(true))));
new DataField(14, "watermark", new BigIntType(true)),
new DataField(
14, "statistics", SerializationUtils.newStringType(true))));

private final FileIO fileIO;
private final Path location;
Expand Down Expand Up @@ -230,8 +234,10 @@ public RecordReader<InternalRow> createReader(Split split) throws IOException {
}
Path location = ((SnapshotsSplit) split).location;
Iterator<Snapshot> snapshots = new SnapshotManager(fileIO, location).snapshots();
StatsFileHandler statsFileHandler = dataTable.store().newStatsFileHandler();
Iterator<InternalRow> rows =
Iterators.transform(snapshots, snapshot -> toRow(snapshot, dataTable));
Iterators.transform(
snapshots, snapshot -> toRow(snapshot, dataTable, statsFileHandler));
if (projection != null) {
rows =
Iterators.transform(
Expand All @@ -240,7 +246,8 @@ public RecordReader<InternalRow> createReader(Split split) throws IOException {
return new IteratorRecordReader<>(rows);
}

private InternalRow toRow(Snapshot snapshot, FileStoreTable dataTable) {
private InternalRow toRow(
Snapshot snapshot, FileStoreTable dataTable, StatsFileHandler statsFileHandler) {
FileStoreScan.Plan plan = dataTable.store().newScan().withSnapshot(snapshot).plan();
return GenericRow.of(
snapshot.id(),
Expand All @@ -260,7 +267,12 @@ private InternalRow toRow(Snapshot snapshot, FileStoreTable dataTable) {
snapshot.changelogRecordCount(),
plan.files(FileKind.ADD).size(),
plan.files(FileKind.DELETE).size(),
snapshot.watermark());
snapshot.watermark(),
statsFileHandler
.readStats(snapshot)
.map(Statistics::toString)
.map(BinaryString::fromString)
.orElse(null));
}
}
}
3 changes: 2 additions & 1 deletion paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ private List<InternalRow> getExceptedResult(long[] snapshotIds) {
snapshot.changelogRecordCount(),
plan.files(FileKind.ADD).size(),
plan.files(FileKind.DELETE).size(),
snapshot.watermark()));
snapshot.watermark(),
null));
}

return expectedRow;
Expand Down
5 changes: 5 additions & 0 deletions ...paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase {
Assertions.assertEquals(2L, stats.mergedRecordCount().getAsLong)
Assertions.assertTrue(stats.mergedRecordSize().isPresent)
Assertions.assertTrue(stats.colStats().isEmpty)

checkAnswer(
spark.sql(
"SELECT commit_kind, substring(statistics, 1, 13) FROM `T$snapshots` WHERE snapshot_id = '3'"),
Row("ANALYZE", "{\n \"snapshot") :: Nil)
}

test("Paimon analyze: analyze no scan") {
Expand Down

0 comments on commit 12b3d73

Please sign in to comment.