Skip to content

Commit

Permalink
[Enhancement] Support move truncated old data to recycle bin (#43107)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Vallishp
Vallishp authored and Your Name committed Nov 28, 2024
1 parent df705ed commit d1773ba
Show file tree
Hide file tree
Showing 25 changed files with 833 additions and 92 deletions.
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,7 @@ unsupportedUseStatement
;

unsupportedDmlStatement
: TRUNCATE TABLE multipartIdentifier specifiedPartition? #truncateTable
: TRUNCATE TABLE multipartIdentifier specifiedPartition? FORCE? #truncateTable
| COPY INTO name=multipartIdentifier columns=identifierList? FROM
(stageAndPattern | (LEFT_PAREN SELECT selectColumnClause
FROM stageAndPattern whereClause? RIGHT_PAREN))
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/cup/sql_parser.cup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7976,9 +7976,9 @@ admin_stmt ::=
;

truncate_stmt ::=
KW_TRUNCATE KW_TABLE base_table_ref:tblRef
KW_TRUNCATE KW_TABLE base_table_ref:tblRef opt_force:force
{:
RESULT = new TruncateTableStmt(tblRef);
RESULT = new TruncateTableStmt(tblRef, force);
:}
;

Expand Down
14 changes: 13 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,21 @@
public class TruncateTableStmt extends DdlStmt implements NotFallbackInParser {

private TableRef tblRef;
private boolean forceDrop;

public TruncateTableStmt(TableRef tblRef) {
public TruncateTableStmt(TableRef tblRef, boolean forceDrop) {
this.tblRef = tblRef;
this.forceDrop = forceDrop;
}

public TableRef getTblRef() {
return tblRef;
}

public boolean isForceDrop() {
return forceDrop;
}

@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
Expand Down Expand Up @@ -75,6 +81,9 @@ public String toSql() {
if (tblRef.getPartitionNames() != null) {
sb.append(tblRef.getPartitionNames().toSql());
}
if (isForceDrop()) {
sb.append(" FORCE");
}
return sb.toString();
}

Expand All @@ -83,6 +92,9 @@ public String toSqlWithoutTable() {
if (tblRef.getPartitionNames() != null) {
sb.append(tblRef.getPartitionNames().toSql());
}
if (isForceDrop()) {
sb.append(" FORCE");
}
return sb.toString();
}

Expand Down
154 changes: 96 additions & 58 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1031,63 +1031,9 @@ private Partition dropPartition(long dbId, String partitionName, boolean isForce
if (partition != null) {
idToPartition.remove(partition.getId());
nameToPartition.remove(partitionName);

if (!isForceDrop) {
// recycle partition
if (partitionInfo.getType() == PartitionType.RANGE) {
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
partitionInfo.getItem(partition.getId()).getItems(),
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));

} else if (partitionInfo.getType() == PartitionType.LIST) {
// construct a dummy range
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);

Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
partitionInfo.getItem(partition.getId()),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));
} else {
// unpartition
// construct a dummy range and dummy list.
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));
}
} else if (!reserveTablets) {
Env.getCurrentEnv().onErasePartition(partition);
}

// drop partition info
partitionInfo.dropPartition(partition.getId());
RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam();
fillInfo(partition, recyclePartitionParam);
dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, reserveTablets);
}
return partition;
}
Expand All @@ -1100,6 +1046,81 @@ public Partition dropPartition(long dbId, String partitionName, boolean isForceD
return dropPartition(dbId, partitionName, isForceDrop, !isForceDrop);
}

private void dropPartitionCommon(long dbId, boolean isForceDrop,
RecyclePartitionParam recyclePartitionParam,
Partition partition,
boolean reserveTablets) {
if (!isForceDrop) {
// recycle partition
if (partitionInfo.getType() == PartitionType.RANGE) {
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
recyclePartitionParam.partitionItem.getItems(),
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);

} else if (partitionInfo.getType() == PartitionType.LIST) {
// construct a dummy range
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);

Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
recyclePartitionParam.partitionItem,
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);
} else {
// unpartition
// construct a dummy range and dummy list.
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);
}
} else if (!reserveTablets) {
Env.getCurrentEnv().onErasePartition(partition);
}

// drop partition info
partitionInfo.dropPartition(partition.getId());
}

public Partition dropPartitionForTruncate(long dbId, boolean isForceDrop,
RecyclePartitionParam recyclePartitionParam) {
// 1. If "isForceDrop" is false, the partition will be added to the Catalog Recyle bin, and all tablets of this
// partition will not be deleted.
// 2. If "ifForceDrop" is true, the partition will be dropped immediately
Partition partition = recyclePartitionParam.partition;
if (partition != null) {
idToPartition.remove(partition.getId());
dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, false);
}
return partition;
}

/*
* A table may contain both formal and temporary partitions.
* There are several methods to get the partition of a table.
Expand Down Expand Up @@ -1918,13 +1939,24 @@ public static OlapTable read(DataInput in) throws IOException {
return GsonUtils.GSON.fromJson(Text.readString(in), OlapTable.class);
}


public void fillInfo(Partition partition, RecyclePartitionParam recyclePartitionParam) {
recyclePartitionParam.dataProperty = partitionInfo.getDataProperty(partition.getId());
recyclePartitionParam.replicaAlloc = partitionInfo.getReplicaAllocation(partition.getId());
recyclePartitionParam.isInMemory = partitionInfo.getIsInMemory(partition.getId());
recyclePartitionParam.isMutable = partitionInfo.getIsMutable(partition.getId());
recyclePartitionParam.partitionItem = partitionInfo.getItem(partition.getId());
recyclePartitionParam.partition = partition;
}

/*
* this method is currently used for truncating table(partitions).
* the new partition has new id, so we need to change all 'id-related' members
*
* return the old partition.
*/
public Partition replacePartition(Partition newPartition) {
public Partition replacePartition(Partition newPartition,
RecyclePartitionParam recyclePartitionParam) {
Partition oldPartition = nameToPartition.remove(newPartition.getName());
idToPartition.remove(oldPartition.getId());

Expand All @@ -1935,6 +1967,12 @@ public Partition replacePartition(Partition newPartition) {
ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(oldPartition.getId());
boolean isInMemory = partitionInfo.getIsInMemory(oldPartition.getId());
boolean isMutable = partitionInfo.getIsMutable(oldPartition.getId());
recyclePartitionParam.dataProperty = dataProperty;
recyclePartitionParam.replicaAlloc = replicaAlloc;
recyclePartitionParam.isInMemory = isInMemory;
recyclePartitionParam.isMutable = isMutable;
recyclePartitionParam.partitionItem = partitionInfo.getItem(oldPartition.getId());
recyclePartitionParam.partition = oldPartition;

if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
Expand Down
31 changes: 31 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.catalog;

public class RecyclePartitionParam {
public Partition partition;
public PartitionItem partitionItem;
public DataProperty dataProperty;
public ReplicaAllocation replicaAlloc;
public boolean isInMemory;
public boolean isMutable = true;

public RecyclePartitionParam() {
// do nothing.
}
}
Loading

0 comments on commit d1773ba

Please sign in to comment.