Load: deserialize PlainDeviceID for TsFile V3 (#231)

apache · Sep 9, 2024 · a90fe9a · a90fe9a
1 parent cd418aa
commit a90fe9a
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 11 deletions.
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/CompatibilityUtils.java
@@ -38,13 +38,20 @@ private CompatibilityUtils() {
   public static DeserializeConfig v3DeserializeConfig = new DeserializeConfig();
 
   static {
+    v3DeserializeConfig.versionNumber =
+        org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER_V3;
     v3DeserializeConfig.tsFileMetadataBufferDeserializer =
         CompatibilityUtils::deserializeTsFileMetadataFromV3;
     v3DeserializeConfig.deviceIDBufferDeserializer =
         ((buffer, context) -> {
           final PlainDeviceID deviceID = PlainDeviceID.deserialize(buffer);
           return deviceID.convertToStringArrayDeviceId();
         });
+    v3DeserializeConfig.deviceIDStreamDeserializer =
+        ((stream, context) -> {
+          final PlainDeviceID deviceID = PlainDeviceID.deserialize(stream);
+          return deviceID.convertToStringArrayDeviceId();
+        });
   }
 
   public static TsFileMetadata deserializeTsFileMetadataFromV3(

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/compatibility/DeserializeConfig.java
@@ -34,6 +34,7 @@
 import java.nio.ByteBuffer;
 
 public class DeserializeConfig {
+  public byte versionNumber = org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER;
   public BufferDeserializer<TsFileMetadata> tsFileMetadataBufferDeserializer =
       TsFileMetadata::deserializeFrom;
 

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java b/java/tsfile/src/main/java/org/apache/tsfile/file/header/ChunkGroupHeader.java
@@ -64,8 +64,8 @@ private int getSerializedSize(IDeviceID deviceID) {
    * @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read ahead.
    * @throws IOException – If an I/O error occurs.
    */
-  public static ChunkGroupHeader deserializeFrom(InputStream inputStream, boolean markerRead)
-      throws IOException {
+  public static ChunkGroupHeader deserializeFrom(
+      InputStream inputStream, boolean markerRead, byte versionNumber) throws IOException {
     if (!markerRead) {
       byte marker = (byte) inputStream.read();
       if (marker != MARKER) {
@@ -74,8 +74,7 @@ public static ChunkGroupHeader deserializeFrom(InputStream inputStream, boolean
     }
 
     // TODO: add an interface in IDeviceID
-    final IDeviceID deviceID =
-        IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+    final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber);
     return new ChunkGroupHeader(deviceID);
   }
 
@@ -85,19 +84,27 @@ public static ChunkGroupHeader deserializeFrom(InputStream inputStream, boolean
    * @param markerRead - Whether the marker of the CHUNK_GROUP_HEADER is read ahead.
    * @throws IOException - If an I/O error occurs.
    */
-  public static ChunkGroupHeader deserializeFrom(TsFileInput input, long offset, boolean markerRead)
-      throws IOException {
+  public static ChunkGroupHeader deserializeFrom(
+      TsFileInput input, long offset, boolean markerRead, byte versionNumber) throws IOException {
     long offsetVar = offset;
     if (!markerRead) {
       offsetVar++;
     }
     input.position(offsetVar);
     final InputStream inputStream = input.wrapAsInputStream();
-    final IDeviceID deviceID =
-        IDeviceID.Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(inputStream);
+    final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber);
     return new ChunkGroupHeader(deviceID);
   }
 
+  private static IDeviceID deserializeDeviceID(InputStream inputStream, byte versionNumber)
+      throws IOException {
+    final IDeviceID.Deserializer deserializer =
+        versionNumber == org.apache.tsfile.common.conf.TSFileConfig.VERSION_NUMBER
+            ? IDeviceID.Deserializer.DEFAULT_DESERIALIZER
+            : IDeviceID.Deserializer.DESERIALIZER_V3;
+    return deserializer.deserializeFrom(inputStream);
+  }
+
   public IDeviceID getDeviceID() {
     return deviceID;
   }

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java
@@ -151,6 +151,7 @@ interface Deserializer {
     IDeviceID deserializeFrom(InputStream inputStream) throws IOException;
 
     Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER();
+    Deserializer DESERIALIZER_V3 = PlainDeviceID.getDESERIALIZER();
   }
 
   interface Factory {

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java
@@ -48,6 +48,23 @@ public PlainDeviceID(String deviceID) {
     this.deviceID = deviceID;
   }
 
+  private static final Deserializer DESERIALIZER =
+      new Deserializer() {
+        @Override
+        public IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
+          return deserialize(byteBuffer).convertToStringArrayDeviceId();
+        }
+
+        @Override
+        public IDeviceID deserializeFrom(InputStream inputStream) throws IOException {
+          return deserialize(inputStream).convertToStringArrayDeviceId();
+        }
+      };
+
+  public static Deserializer getDESERIALIZER() {
+    return DESERIALIZER;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -1479,7 +1479,8 @@ protected Pair<IMetadataIndexEntry, Long> getMetadataAndEndOffsetOfMeasurementNo
    * @throws IOException io error
    */
   public ChunkGroupHeader readChunkGroupHeader() throws IOException {
-    return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true);
+    return ChunkGroupHeader.deserializeFrom(
+        tsFileInput.wrapAsInputStream(), true, deserializeConfig.versionNumber);
   }
 
   /**
@@ -1492,7 +1493,8 @@ public ChunkGroupHeader readChunkGroupHeader() throws IOException {
    */
   public ChunkGroupHeader readChunkGroupHeader(long position, boolean markerRead)
       throws IOException {
-    return ChunkGroupHeader.deserializeFrom(tsFileInput, position, markerRead);
+    return ChunkGroupHeader.deserializeFrom(
+        tsFileInput, position, markerRead, deserializeConfig.versionNumber);
   }
 
   public void readPlanIndex() throws IOException {

diff --git a/.../src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/.../src/main/java/org/apache/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -59,7 +59,7 @@ public TsFileSequenceReaderTimeseriesMetadataIterator(
     this.reader = reader;
     this.needChunkMetadata = needChunkMetadata;
     this.timeseriesBatchReadNumber = timeseriesBatchReadNumber;
-    this.deserializeConfig = new DeserializeConfig();
+    this.deserializeConfig = this.reader.getDeserializeContext();
 
     if (this.reader.tsFileMetaData == null) {
       this.reader.readFileMetadata();