Skip to content

Commit

Permalink
add ColStats<T extends Comparable<T>>
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zouxxyy
Zouxxyy committed Jan 12, 2024
1 parent 0391cf8 commit d162d81
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 21 deletions.
18 changes: 9 additions & 9 deletions paimon-core/src/main/java/org/apache/paimon/stats/ColStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
* <li>maxLen: max column length
* </ul>
*/
public class ColStats {
public class ColStats<T extends Comparable<T>> {

private static final String FIELD_DISTINCT_COUNT = "distinctCount";
private static final String FIELD_MIN = "min";
Expand All @@ -62,13 +62,13 @@ public class ColStats {
@JsonProperty(FIELD_MIN)
private @Nullable String serializedMin;

private @Nullable Object min;
private @Nullable T min;

@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty(FIELD_MAX)
private @Nullable String serializedMax;

private @Nullable Object max;
private @Nullable T max;

@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonProperty(FIELD_NULL_COUNT)
Expand Down Expand Up @@ -100,8 +100,8 @@ public ColStats(

public ColStats(
@Nullable Long distinctCount,
@Nullable Object min,
@Nullable Object max,
@Nullable T min,
@Nullable T max,
@Nullable Long nullCount,
@Nullable Long avgLen,
@Nullable Long maxLen) {
Expand All @@ -117,11 +117,11 @@ public OptionalLong distinctCount() {
return OptionalUtils.ofNullable(distinctCount);
}

public Optional<Object> min() {
public Optional<T> min() {
return Optional.ofNullable(min);
}

public Optional<Object> max() {
public Optional<T> max() {
return Optional.ofNullable(max);
}

Expand Down Expand Up @@ -151,7 +151,7 @@ public void serializeFieldsToString(DataType dataType) {

public void deserializeFieldsFromString(DataType dataType) {
if ((serializedMin != null && min == null) || (serializedMax != null && max == null)) {
Serializer<Object> serializer = InternalSerializers.create(dataType);
Serializer<T> serializer = InternalSerializers.create(dataType);
if (serializedMin != null && min == null) {
min = serializer.deserializeFromString(serializedMin);
}
Expand All @@ -169,7 +169,7 @@ public boolean equals(Object object) {
if (object == null || getClass() != object.getClass()) {
return false;
}
ColStats colStats = (ColStats) object;
ColStats<?> colStats = (ColStats<?>) object;
return Objects.equals(distinctCount, colStats.distinctCount)
&& Objects.equals(serializedMin, colStats.serializedMin)
&& Objects.equals(min, colStats.min)
Expand Down
14 changes: 7 additions & 7 deletions paimon-core/src/main/java/org/apache/paimon/stats/Stats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@ public class Stats {
private final @Nullable Long mergedRecordSize;

@JsonProperty(FIELD_COL_STATS)
private final Map<String, ColStats> colStats;
private final Map<String, ColStats<?>> colStats;

@JsonCreator
public Stats(
@JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId,
@JsonProperty(FIELD_MERGED_RECORD_COUNT) @Nullable Long mergedRecordCount,
@JsonProperty(FIELD_MERGED_RECORD_SIZE) @Nullable Long mergedRecordSize,
@JsonProperty(FIELD_COL_STATS) Map<String, ColStats> colStats) {
@JsonProperty(FIELD_COL_STATS) Map<String, ColStats<?>> colStats) {
this.snapshotId = snapshotId;
this.mergedRecordCount = mergedRecordCount;
this.mergedRecordSize = mergedRecordSize;
Expand All @@ -95,16 +95,16 @@ public OptionalLong mergedRecordSize() {
return OptionalUtils.ofNullable(mergedRecordSize);
}

public Map<String, ColStats> colStats() {
public Map<String, ColStats<?>> colStats() {
return colStats;
}

public void serializeFieldsToString(TableSchema schema) {
try {
if (colStats != null) {
for (Map.Entry<String, ColStats> entry : colStats.entrySet()) {
for (Map.Entry<String, ColStats<?>> entry : colStats.entrySet()) {
String colName = entry.getKey();
ColStats colStats = entry.getValue();
ColStats<?> colStats = entry.getValue();
DataType type =
schema.fields().stream()
.filter(field -> field.name().equals(colName))
Expand All @@ -122,9 +122,9 @@ public void serializeFieldsToString(TableSchema schema) {
public void deserializeFieldsFromString(TableSchema schema) {
try {
if (colStats != null) {
for (Map.Entry<String, ColStats> entry : colStats.entrySet()) {
for (Map.Entry<String, ColStats<?>> entry : colStats.entrySet()) {
String colName = entry.getKey();
ColStats colStats = entry.getValue();
ColStats<?> colStats = entry.getValue();
DataType type =
schema.fields().stream()
.filter(field -> field.name().equals(colName))
Expand Down
6 changes: 3 additions & 3 deletions paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,8 +788,8 @@ public void testWriteStats() throws Exception {
store.commitData(generateDataList(10), gen::getPartition, kv -> 0, Collections.emptyMap());

// Analyze and check
HashMap<String, ColStats> fakeColStatsMap = new HashMap<>();
fakeColStatsMap.put("orderId", new ColStats(10L, 1L, 10L, 0L, 8L, 8L));
HashMap<String, ColStats<?>> fakeColStatsMap = new HashMap<>();
fakeColStatsMap.put("orderId", new ColStats<>(10L, 1L, 10L, 0L, 8L, 8L));
Stats fakeStats =
new Stats(store.snapshotManager().latestSnapshotId(), 10L, 1000L, fakeColStatsMap);
fileStoreCommit.commitStatistics(fakeStats, Long.MAX_VALUE);
Expand All @@ -814,7 +814,7 @@ public void testWriteStats() throws Exception {

// Then we need to analyze again
fakeColStatsMap = new HashMap<>();
fakeColStatsMap.put("orderId", new ColStats(30L, 1L, 30L, 0L, 8L, 8L));
fakeColStatsMap.put("orderId", new ColStats<>(30L, 1L, 30L, 0L, 8L, 8L));
fakeStats =
new Stats(store.snapshotManager().latestSnapshotId(), 30L, 3000L, fakeColStatsMap);
fileStoreCommit.commitStatistics(fakeStats, Long.MAX_VALUE);
Expand Down
4 changes: 2 additions & 2 deletions paimon-core/src/test/java/org/apache/paimon/stats/StatsFileTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ public Path toPath(String fileName) {
};

StatsFile file = new StatsFile(LocalFileIO.create(), pathFactory);
HashMap<String, ColStats> colStatsMap = new HashMap<>();
colStatsMap.put("orderId", new ColStats(10L, "111", "222", 0L, 8L, 8L));
HashMap<String, ColStats<?>> colStatsMap = new HashMap<>();
colStatsMap.put("orderId", new ColStats<>(10L, "111", "222", 0L, 8L, 8L));
Stats stats = new Stats(1L, 10L, 1000L, colStatsMap);
String fileName = file.write(stats);

Expand Down

0 comments on commit d162d81

Please sign in to comment.