-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b5e0f2c
commit f6274ef
Showing
13 changed files
with
451 additions
and
242 deletions.
There are no files selected for viewing
95 changes: 95 additions & 0 deletions
95
...on-flink-common/src/main/java/org/apache/paimon/flink/compact/AbstractTableScanLogic.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
|
|
||
| package org.apache.paimon.flink.compact; | ||
|
|
||
| import org.apache.paimon.catalog.Catalog; | ||
| import org.apache.paimon.catalog.Identifier; | ||
| import org.apache.paimon.table.BucketMode; | ||
| import org.apache.paimon.table.FileStoreTable; | ||
| import org.apache.paimon.table.Table; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.util.List; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| import static org.apache.paimon.flink.utils.MultiTablesCompactorUtil.shouldCompactTable; | ||
|
|
||
| public abstract class AbstractTableScanLogic <T> implements CompactionTableScanner.TableScanLogic<T> { | ||
| private static final Logger LOG = LoggerFactory.getLogger(AbstractTableScanLogic.class); | ||
| protected final Catalog.Loader catalogLoader; | ||
| protected final Pattern includingPattern; | ||
| protected final Pattern excludingPattern; | ||
| protected final Pattern databasePattern; | ||
|
|
||
| protected transient Catalog catalog; | ||
|
|
||
| protected AtomicBoolean isRunning; | ||
| protected boolean isStreaming; | ||
|
|
||
| public AbstractTableScanLogic(Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, Pattern databasePattern, AtomicBoolean isRunning, boolean isStreaming) { | ||
| this.catalogLoader = catalogLoader; | ||
| catalog = catalogLoader.load(); | ||
|
|
||
| this.includingPattern = includingPattern; | ||
| this.excludingPattern = excludingPattern; | ||
| this.databasePattern = databasePattern; | ||
| this.isRunning = isRunning; | ||
| this.isStreaming = isStreaming; | ||
| } | ||
|
|
||
| protected void updateTableMap() | ||
| throws Catalog.DatabaseNotExistException, Catalog.TableNotExistException { | ||
| List<String> databases = catalog.listDatabases(); | ||
|
|
||
| for (String databaseName : databases) { | ||
| if (databasePattern.matcher(databaseName).matches()) { | ||
| List<String> tables = catalog.listTables(databaseName); | ||
| for (String tableName : tables) { | ||
| Identifier identifier = Identifier.create(databaseName, tableName); | ||
| if (shouldCompactTable(identifier, includingPattern, excludingPattern) | ||
| && (!tableScanned(identifier))) { | ||
| Table table = catalog.getTable(identifier); | ||
| if (!(table instanceof FileStoreTable)) { | ||
| LOG.error( | ||
| String.format( | ||
| "Only FileStoreTable supports compact action. The table type is '%s'.", | ||
| table.getClass().getName())); | ||
| continue; | ||
| } | ||
|
|
||
| FileStoreTable fileStoreTable = (FileStoreTable) table; | ||
| if (fileStoreTable.bucketMode() == BucketMode.UNAWARE) { | ||
| LOG.info( | ||
| String.format( | ||
| "the bucket mode of %s is unware. ", | ||
| identifier.getFullName()) | ||
| + "currently, the table with unware bucket mode is not support in combined mode."); | ||
| continue; | ||
| } | ||
|
|
||
| addScanTable(fileStoreTable, identifier); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
45 changes: 45 additions & 0 deletions
45
.../paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/BatchTableScanner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.flink.compact; | ||
|
|
||
| import org.apache.flink.streaming.api.functions.source.SourceFunction; | ||
|
|
||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| public class BatchTableScanner<T> implements CompactionTableScanner<T> { | ||
| private AtomicBoolean isRunning = new AtomicBoolean(true); | ||
| private TableScanLogic<T> tableScanLogic; | ||
|
|
||
| public BatchTableScanner(AtomicBoolean isRunning, TableScanLogic<T> tableScanLogic) { | ||
| this.isRunning = isRunning; | ||
| this.tableScanLogic = tableScanLogic; | ||
| } | ||
|
|
||
| @Override | ||
| public void scan(SourceFunction.SourceContext<T> ctx) throws Exception { | ||
| if (isRunning.get()) { | ||
| Boolean isEmpty = tableScanLogic.collectFiles(ctx); | ||
| if (isEmpty == null) return; | ||
| if (isEmpty) { | ||
| throw new Exception( | ||
| "No file were collected. Please ensure there are tables detected after pattern matching"); | ||
| } | ||
| } | ||
| } | ||
| } |
39 changes: 39 additions & 0 deletions
39
...on-flink-common/src/main/java/org/apache/paimon/flink/compact/CompactionTableScanner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.flink.compact; | ||
|
|
||
| import org.apache.flink.streaming.api.functions.source.SourceFunction; | ||
| import org.apache.paimon.catalog.Catalog; | ||
| import org.apache.paimon.catalog.Identifier; | ||
| import org.apache.paimon.table.FileStoreTable; | ||
|
|
||
| public interface CompactionTableScanner<T> { | ||
| void scan(SourceFunction.SourceContext<T> ctx) throws Exception; | ||
|
|
||
| interface TableScanLogic<T>{ | ||
| Boolean collectFiles(SourceFunction.SourceContext<T> ctx) throws Catalog.TableNotExistException, Catalog.DatabaseNotExistException; | ||
|
|
||
| boolean tableScanned(Identifier identifier); | ||
|
|
||
| void addScanTable(FileStoreTable fileStoreTable, Identifier identifier); | ||
| } | ||
| // | ||
|
|
||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
...mon-flink-common/src/main/java/org/apache/paimon/flink/compact/StreamingTableScanner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.flink.compact; | ||
|
|
||
| import org.apache.flink.streaming.api.functions.source.SourceFunction; | ||
|
|
||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| public class StreamingTableScanner<T> implements CompactionTableScanner<T> { | ||
| private final AtomicBoolean isRunning; | ||
|
|
||
| private final long monitorInterval; | ||
|
|
||
| private final TableScanLogic<T> tableScanLogic; | ||
|
|
||
| public StreamingTableScanner(long monitorInterval, TableScanLogic<T> tableScanLogic, AtomicBoolean isRunning) { | ||
| this.monitorInterval = monitorInterval; | ||
| this.tableScanLogic = tableScanLogic; | ||
| this.isRunning = isRunning; | ||
| } | ||
|
|
||
| @SuppressWarnings("BusyWait") | ||
| @Override | ||
| public void scan(SourceFunction.SourceContext<T> ctx) throws Exception { | ||
| while (isRunning.get()) { | ||
| Boolean isEmpty = tableScanLogic.collectFiles(ctx); | ||
| if (isEmpty == null) return; | ||
| if (isEmpty) { | ||
| Thread.sleep(monitorInterval); | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.