From 770165a1cb9744bf899267dacb05db5a0718bb31 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Mon, 25 Nov 2024 11:20:09 +0800 Subject: [PATCH] Implement automatic conversion of aggFun under table model. --- .../iot/benchmark/conf/ConfigDescriptor.java | 48 ++++++++++++++++++- .../iot/benchmark/conf/Constants.java | 18 +++++++ .../iot/benchmark/iotdb200/IoTDB.java | 8 ++-- .../ModelStrategy/IoTDBModelStrategy.java | 17 +++++-- .../iotdb200/ModelStrategy/TableStrategy.java | 32 ++++++++++--- .../iotdb200/ModelStrategy/TreeStrategy.java | 23 +++++++-- 6 files changed, 124 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 3b77fd144..a89e07371 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -456,8 +456,8 @@ private void loadProps() { config.setQUERY_DEVICE_NUM( Integer.parseInt( properties.getProperty("QUERY_DEVICE_NUM", config.getQUERY_DEVICE_NUM() + ""))); - config.setQUERY_AGGREGATE_FUN( - properties.getProperty("QUERY_AGGREGATE_FUN", config.getQUERY_AGGREGATE_FUN())); + + loadAndConvertAggregateFunction(properties); config.setQUERY_INTERVAL( Long.parseLong( @@ -553,6 +553,38 @@ private void loadProps() { } } + /** for table model */ + private void loadAndConvertAggregateFunction(Properties properties) { + String aggFun = properties.getProperty("QUERY_AGGREGATE_FUN", config.getQUERY_AGGREGATE_FUN()); + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + switch (aggFun) { + case Constants.MAX_TIME: + config.setQUERY_AGGREGATE_FUN(Constants.LAST_BY); + break; + case Constants.MIN_TIME: + config.setQUERY_AGGREGATE_FUN(Constants.FIRST_BY); + break; + case Constants.MAX_VALUE: + config.setQUERY_AGGREGATE_FUN(Constants.MAX_AGG); + break; + case Constants.MIN_VALUE: + config.setQUERY_AGGREGATE_FUN(Constants.MIN_AGG); + break; + case Constants.FIRST_VALUE: + config.setQUERY_AGGREGATE_FUN(Constants.FIRST); + break; + case Constants.LAST_VALUE: + config.setQUERY_AGGREGATE_FUN(Constants.LAST); + break; + case Constants.TIME_DURATION: + config.setQUERY_AGGREGATE_FUN(Constants.TIME_DURATION); + break; + } + } else { + config.setQUERY_AGGREGATE_FUN(aggFun); + } + } + /** Check validation of config */ private boolean checkConfig() { boolean result = true; @@ -634,6 +666,18 @@ private boolean checkConfig() { "The iotdb table model only supports INSERT_USE_SESSION_TABLET! Please modify DB_SWITCH in the configuration file."); result = false; } + // TODO Not supported TIME_DURATION、MAX_BY、MIN_BY. iotdb will report errors for these three + // types of aggFun. + if (config.getQUERY_AGGREGATE_FUN().equals(Constants.MAX_BY) + || config.getQUERY_AGGREGATE_FUN().equals(Constants.MIN_BY)) { + LOGGER.error("MAX_BY or MIN_BY not yet supported !"); + result = false; + } + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE + && config.getQUERY_AGGREGATE_FUN().equals(Constants.TIME_DURATION)) { + LOGGER.error("TIME_DURATION not yet supported !"); + result = false; + } result &= checkInsertDataTypeProportion(); result &= checkOperationProportion(); if (config.getSCHEMA_CLIENT_NUMBER() == 0 || config.getDATA_CLIENT_NUMBER() == 0) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java index 4cf16c75b..1a80b1f99 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java @@ -91,4 +91,22 @@ public class Constants { public static final String PI_ARCHIVE_CLASS = "cn.edu.tsinghua.iot.benchmark.piarchive.PIArchive"; public static final String IGINX_CLASS = "cn.edu.tsinghua.iot.benchmark.iginx.IginX"; public static final String SELF_CHECK_CLASS = "cn.edu.tsinghua.iot.benchmark.tsdb.self.SelfCheck"; + + public static final String MAX_TIME = "max_time"; + public static final String MIN_TIME = "min_time"; + public static final String TIME_DURATION = "time_duration"; + public static final String MAX_VALUE = "max_value"; + public static final String MIN_VALUE = "min_value"; + public static final String FIRST_VALUE = "first_value"; + public static final String LAST_VALUE = "last_value"; + + public static final String LAST_BY = "last_by"; + public static final String FIRST_BY = "first_by"; + public static final String MAX_AGG = "max"; + public static final String MIN_AGG = "min"; + public static final String LAST = "last"; + public static final String FIRST = "first"; + public static final String COUNT = "count"; + public static final String MAX_BY = "max_by"; + public static final String MIN_BY = "min_by"; } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 9c79e3623..be44d83b5 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -362,7 +362,7 @@ public Status aggRangeValueQuery(AggRangeValueQuery aggRangeValueQuery) { */ @Override public Status groupByQuery(GroupByQuery groupByQuery) { - String sql = modelStrategy.getGroupByQuerySQL(groupByQuery); + String sql = modelStrategy.getGroupByQuerySQL(groupByQuery, false); return executeQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY); } @@ -400,7 +400,7 @@ public Status rangeQueryOrderByDesc(RangeQuery rangeQuery) { 0, builder); // ORDER BY - builder.append(" ORDER BY time desc"); + modelStrategy.addOrderByTimeDesc(builder); return executeQueryAndGetStatus(builder.toString(), Operation.RANGE_QUERY_ORDER_BY_TIME_DESC); } @@ -425,7 +425,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { (int) valueRangeQuery.getValueThreshold(), builder); // ORDER BY - builder.append(" ORDER BY time desc"); + modelStrategy.addOrderByTimeDesc(builder); return executeQueryAndGetStatus( builder.toString(), Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); } @@ -439,7 +439,7 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { */ @Override public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { - String sql = modelStrategy.getGroupByQuerySQL(groupByQuery); + String sql = modelStrategy.getGroupByQuerySQL(groupByQuery, true); return executeQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC); } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java index 2503c21ed..2270dce03 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java @@ -25,6 +25,7 @@ import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; +import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; @@ -67,12 +68,14 @@ public abstract void registerSchema( public abstract String getAggQuerySqlHead(List devices, String aggFun); - public abstract String getGroupByQuerySQL(GroupByQuery groupByQuery); + public abstract String getGroupByQuerySQL(GroupByQuery groupByQuery, Boolean addOrderBy); public abstract String getLatestPointQuerySql(List devices); public abstract void addFromClause(List devices, StringBuilder builder); + public abstract void addOrderByTimeDesc(StringBuilder builder); + public abstract void addPreciseQueryWhereClause( String strTime, List deviceSchemas, StringBuilder builder); @@ -164,12 +167,18 @@ public abstract void sessionCleanupImpl(Session session) protected String getAggFunForGroupByQuery(List querySensors, String aggFunction) { StringBuilder builder = new StringBuilder(); - builder.append(aggFunction).append("(").append(querySensors.get(0).getName()).append(")"); - for (int i = 1; i < querySensors.size(); i++) { + String timeArg = + (Constants.LAST_BY.equals(aggFunction) || Constants.FIRST_BY.equals(aggFunction)) + ? "time, " + : ""; + for (int i = 0; i < querySensors.size(); i++) { + if (i > 0) { + builder.append(", "); + } builder - .append(", ") .append(aggFunction) .append("(") + .append(timeArg) .append(querySensors.get(i).getName()) .append(")"); } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 4077f40dd..8d6cc1251 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -24,6 +24,7 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; +import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; @@ -167,16 +168,31 @@ public void addFromClause(List devices, StringBuilder builder) { .append(devices.get(0).getTable()); } + @Override + public void addOrderByTimeDesc(StringBuilder builder) { + builder.append(" ORDER BY device_id, time desc"); + } + @Override public String getAggQuerySqlHead(List devices, String aggFun) { StringBuilder builder = new StringBuilder(); builder.append("SELECT device_id"); List querySensors = devices.get(0).getSensors(); - for (int i = 1; i < querySensors.size(); i++) { + String timeArg = + (Constants.FIRST_BY.equals(aggFun) + || Constants.LAST_BY.equals(aggFun) + || Constants.MAX_BY.equals(aggFun) + || Constants.MIN_BY.equals(aggFun)) + ? "time, " + : ""; + for (int i = 0; i < querySensors.size(); i++) { + if (i > 0) { + builder.append(", "); + } builder - .append(", ") .append(aggFun) .append("(") + .append(timeArg) .append(querySensors.get(i).getName()) .append(")"); } @@ -192,7 +208,7 @@ public String getAggQuerySqlHead(List devices, String aggFun) { *

getAggForGroupByQuery */ @Override - public String getGroupByQuerySQL(GroupByQuery groupByQuery) { + public String getGroupByQuerySQL(GroupByQuery groupByQuery, Boolean addOrderBy) { StringBuilder builder = new StringBuilder(); // SELECT builder @@ -219,10 +235,12 @@ public String getGroupByQuerySQL(GroupByQuery groupByQuery) { .append(groupByQuery.getGranularity()) .append("ms, time)"); // ORDER BY - builder - .append(" order by device_id, date_bin(") - .append(groupByQuery.getGranularity()) - .append("ms, time) desc"); + if (addOrderBy) { + builder + .append(" order by device_id, date_bin(") + .append(groupByQuery.getGranularity()) + .append("ms, time) desc"); + } return builder.toString(); } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java index d5a4fe4b9..9ec61f77b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java @@ -25,6 +25,7 @@ import org.apache.iotdb.session.Session; import org.apache.iotdb.session.template.MeasurementNode; +import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.entity.Batch.IBatch; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; @@ -221,12 +222,17 @@ public String getAggQuerySqlHead(List devices, String aggFun) { StringBuilder builder = new StringBuilder(); builder.append("SELECT "); List querySensors = devices.get(0).getSensors(); - builder.append(aggFun).append("(").append(querySensors.get(0).getName()).append(")"); - for (int i = 1; i < querySensors.size(); i++) { + String timeArg = + (Constants.MAX_BY.equals(aggFun) || Constants.MIN_BY.equals(aggFun)) ? "time, " : ""; + + for (int i = 0; i < querySensors.size(); i++) { + if (i > 0) { + builder.append(", "); + } builder - .append(", ") .append(aggFun) .append("(") + .append(timeArg) .append(querySensors.get(i).getName()) .append(")"); } @@ -243,7 +249,12 @@ public void addFromClause(List devices, StringBuilder builder) { } @Override - public String getGroupByQuerySQL(GroupByQuery groupByQuery) { + public void addOrderByTimeDesc(StringBuilder builder) { + builder.append(" ORDER BY time desc"); + } + + @Override + public String getGroupByQuerySQL(GroupByQuery groupByQuery, Boolean addOrderBy) { StringBuilder builder = new StringBuilder(); // SELECT builder @@ -260,7 +271,9 @@ public String getGroupByQuerySQL(GroupByQuery groupByQuery) { groupByQuery.getEndTimestamp(), groupByQuery.getGranularity()); // ORDER BY - builder.append(" ORDER BY time desc"); + if (addOrderBy) { + builder.append(" ORDER BY time desc"); + } return builder.toString(); }