Dev idtable

server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java

@@ -25,7 +25,7 @@
 import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
 import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup;
 import org.apache.iotdb.db.exception.runtime.FlushRunTimeException;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 import org.apache.iotdb.db.rescon.SystemInfo;
 import org.apache.iotdb.db.service.metrics.MetricService;
 import org.apache.iotdb.db.service.metrics.enums.Metric;

server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java

@@ -26,8 +26,8 @@
 import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
 import org.apache.iotdb.db.exception.WriteProcessException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;

server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java

@@ -25,7 +25,7 @@
 import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
 import org.apache.iotdb.db.exception.WriteProcessException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
 import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;

server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java

@@ -19,7 +19,7 @@
 
 package org.apache.iotdb.db.engine.memtable;
 
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 
 import java.util.HashMap;
 import java.util.Map;

server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java

@@ -44,8 +44,8 @@
 import org.apache.iotdb.db.exception.WriteProcessException;
 import org.apache.iotdb.db.exception.WriteProcessRejectException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.path.AlignedPath;
 import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;

server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java

@@ -21,6 +21,7 @@
 
 import org.apache.iotdb.commons.exception.MetadataException;
 import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IStatefulDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
@@ -38,6 +39,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -148,7 +150,6 @@ public void recover(IDTable idTable) {
     try (FileInputStream inputStream = new FileInputStream(dataFile)) {
       // read file version
       ReadWriteIOUtils.readString(inputStream);
-
       while (inputStream.available() > 0) {
         DiskSchemaEntry cur = DiskSchemaEntry.deserialize(inputStream);
         if (!cur.deviceID.equals(DiskSchemaEntry.TOMBSTONE)) {
@@ -158,11 +159,30 @@ public void recover(IDTable idTable) {
                   TSEncoding.deserialize(cur.encoding),
                   CompressionType.deserialize(cur.compressor),
                   loc);
+          // if the system uses IStatefulDeviceID, need to recover the state of device id
+          if (IStatefulDeviceID.class.isAssignableFrom(
+              DeviceIDFactory.getInstance().getDeviceIDClass())) {
+            IStatefulDeviceID statefulIDeviceID =
+                (IStatefulDeviceID)
+                    DeviceIDFactory.getInstance()
+                        .getDeviceIDClass()
+                        .getDeclaredConstructor()
+                        .newInstance();
+            statefulIDeviceID.recover(
+                cur.seriesKey.substring(
+                    0, cur.seriesKey.length() - cur.measurementName.length() - 1),
+                cur.deviceID);
+          }
           idTable.putSchemaEntry(cur.deviceID, cur.measurementName, schemaEntry, cur.isAligned);
         }
         loc += cur.entrySize;
       }
-    } catch (IOException | MetadataException e) {
+    } catch (IOException
+        | MetadataException
+        | IllegalAccessException
+        | InstantiationException
+        | NoSuchMethodException
+        | InvocationTargetException e) {
       logger.info("Last entry is incomplete, we will recover as much as we can.");
       try {
         outputStream.getChannel().truncate(loc);

server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java

@@ -24,10 +24,10 @@
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID;
 import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
@@ -133,6 +133,7 @@ void updateLastCache(
       throws MetadataException;
 
   /** clear id table and close file */
+  @TestOnly
   void clear() throws IOException;
 
   /**
@@ -143,6 +144,14 @@ void updateLastCache(
    */
   DeviceEntry getDeviceEntry(String deviceName);
 
+  /**
+   * get device entry from device id
+   *
+   * @param deviceID device id of the device path
+   * @return device entry
+   */
+  DeviceEntry getDeviceEntry(IDeviceID deviceID);
+
   /**
    * get schema from device and measurements
    *
@@ -162,13 +171,13 @@ void updateLastCache(
   /**
    * put schema entry to id table, currently used in recover
    *
-   * @param devicePath device path (can be device id formed path)
+   * @param deviceID device id
    * @param measurement measurement name
    * @param schemaEntry schema entry to put
    * @param isAligned is the device aligned
    */
   void putSchemaEntry(
-      String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned)
+      String deviceID, String measurement, SchemaEntry schemaEntry, boolean isAligned)
       throws MetadataException;
 
   /**

server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java

@@ -25,10 +25,11 @@
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
+import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID;
+import org.apache.iotdb.db.metadata.idtable.deviceID.StandAloneAutoIncDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
 import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry;
-import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
 import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode;
 import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry;
 import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID;
@@ -72,6 +73,7 @@ public class IDTableHashmapImpl implements IDTable {
 
   /** disk schema manager to manage disk schema entry */
   private IDiskSchemaManager IDiskSchemaManager;
+
   /** iotdb config */
   protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
 
@@ -307,10 +309,14 @@ public synchronized void updateLastCache(
   }
 
   @Override
+  @TestOnly
   public void clear() throws IOException {
     if (IDiskSchemaManager != null) {
       IDiskSchemaManager.close();
     }
+    if (DeviceIDFactory.getInstance().getDeviceIDClass() == StandAloneAutoIncDeviceID.class) {
+      StandAloneAutoIncDeviceID.clear();
+    }
   }
 
   /**
@@ -321,7 +327,17 @@ public void clear() throws IOException {
    */
   @Override
   public DeviceEntry getDeviceEntry(String deviceName) {
-    IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName);
+    return getDeviceEntry(DeviceIDFactory.getInstance().getDeviceID(deviceName));
+  }
+
+  /**
+   * get device entry from deviceID
+   *
+   * @param deviceID deviceID of the device path
+   * @return device entry
+   */
+  @Override
+  public DeviceEntry getDeviceEntry(IDeviceID deviceID) {
     int slot = calculateSlot(deviceID);
 
     // reuse device entry in map
@@ -355,6 +371,11 @@ public IMeasurementSchema getSeriesSchema(String deviceName, String measurementN
         schemaEntry.getCompressionType());
   }
 
+  /**
+   * get all useful deviceEntry of idTable
+   *
+   * @return device entries
+   */
   @Override
   public List<DeviceEntry> getAllDeviceEntry() {
     List<DeviceEntry> res = new ArrayList<>();
@@ -365,11 +386,20 @@ public List<DeviceEntry> getAllDeviceEntry() {
     return res;
   }
 
+  /**
+   * put schema entry to id table, currently used in recover
+   *
+   * @param deviceID device id
+   * @param measurement measurement name
+   * @param schemaEntry schema entry to put
+   * @param isAligned is the device aligned
+   * @throws MetadataException
+   */
   @Override
   public void putSchemaEntry(
-      String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned)
+      String deviceID, String measurement, SchemaEntry schemaEntry, boolean isAligned)
       throws MetadataException {
-    DeviceEntry deviceEntry = getDeviceEntryWithAlignedCheck(devicePath, isAligned);
+    DeviceEntry deviceEntry = getDeviceEntryWithAlignedCheck(deviceID, isAligned);
     deviceEntry.putSchemaEntry(measurement, schemaEntry);
   }
 
@@ -460,7 +490,7 @@ private IMeasurementMNode getOrCreateMeasurementIfNotExist(
    */
   private DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean isAligned)
       throws MetadataException {
-    IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName);
+    IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(deviceName);
     int slot = calculateSlot(deviceID);
 
     DeviceEntry deviceEntry = idTables[slot].get(deviceID);

server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java

@@ -21,6 +21,7 @@
 import org.apache.iotdb.commons.exception.MetadataException;
 import org.apache.iotdb.commons.file.SystemFileFactory;
 import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.exception.metadata.PathNotExistException;
 import org.apache.iotdb.db.service.IoTDB;
@@ -127,6 +128,7 @@ public synchronized IMeasurementSchema getSeriesSchema(String deviceName, String
   }
 
   /** clear id table map */
+  @TestOnly
   public void clear() throws IOException {
     for (IDTable idTable : idTableMap.values()) {
       idTable.clear();

server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java → server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.db.metadata.idtable.entry;
+package org.apache.iotdb.db.metadata.idtable.deviceID;
 
 import java.nio.ByteBuffer;
 
@@ -29,7 +29,7 @@ public interface IDeviceID {
    *
    * @return string format device id
    */
-  public String toStringID();
+  String toStringID();
 
-  public void serialize(ByteBuffer byteBuffer);
+  void serialize(ByteBuffer byteBuffer);
 }

server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IStatefulDeviceID.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.metadata.idtable.deviceID;
+
+/** stateful device id interface */
+public interface IStatefulDeviceID extends IDeviceID {
+
+  /**
+   * recover state using devicePath and device id
+   *
+   * @param devicePath device path of the time series
+   * @param deviceID device id
+   */
+  void recover(String devicePath, String deviceID);
+}

server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java → server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.db.metadata.idtable.entry;
+package org.apache.iotdb.db.metadata.idtable.deviceID;
 
 import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
 

server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java → server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.iotdb.db.metadata.idtable.entry;
+package org.apache.iotdb.db.metadata.idtable.deviceID;
 
 import org.apache.iotdb.db.metadata.idtable.IDTable;
 import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -90,7 +90,7 @@ private void fromSHA256String(String deviceID) {
    *
    * @param deviceID device path
    */
-  private void buildSHA256(String deviceID) {
+  protected void buildSHA256(String deviceID) {
     byte[] hashVal;
     int slot = calculateSlot(deviceID);