From 1c13bb82c5d407f5f69a37fb3f84b5d0a7dadca8 Mon Sep 17 00:00:00 2001 From: Ken Wenzel Date: Wed, 22 May 2024 18:40:38 +0200 Subject: [PATCH] Disable gc for LMDB store and use correct uptime of JVM. --- .../java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java | 9 ++++++--- .../main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java | 1 + .../org/eclipse/rdf4j/sail/lmdb/MemoryOverflowModel.java | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java index e643860537..b632bf792b 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java @@ -73,6 +73,7 @@ class LmdbSailStore implements SailStore { private volatile boolean nextTransactionAsync; boolean enableMultiThreading = true; + boolean enableGc = true; private PersistentSetFactory setFactory; private PersistentSet unusedIds, nextUnusedIds; @@ -760,9 +761,11 @@ private long removeStatements(long subj, long pred, long obj, boolean explicit, for (long contextId : contexts) { tripleStore.removeTriplesByContext(subj, pred, obj, contextId, explicit, quad -> { removeCount[0]++; - for (long id : quad) { - if (id != 0L) { - unusedIds.add(id); + if (enableGc) { + for (long id : quad) { + if (id != 0L) { + unusedIds.add(id); + } } } }); diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java index eab78afaaa..d64c5ffbe0 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java @@ -258,6 +258,7 @@ protected LmdbSailStore createSailStore(File dataDir) throws IOException, SailEx LmdbStoreConfig overflowConfig = new LmdbStoreConfig(); LmdbSailStore store = new LmdbSailStore(dataDir, overflowConfig); store.enableMultiThreading = false; + store.enableGc = false; // does not need to isolate transactions and therefore can optimize autogrow and others store.setTransactionIsolation(false); return store; diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/MemoryOverflowModel.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/MemoryOverflowModel.java index 5199a71ada..0084706dba 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/MemoryOverflowModel.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/MemoryOverflowModel.java @@ -17,6 +17,7 @@ import java.io.ObjectOutputStream; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.nio.file.Files; import java.util.Collection; import java.util.HashSet; @@ -88,11 +89,13 @@ abstract class MemoryOverflowModel extends AbstractModel implements AutoCloseabl private static volatile List gcInfos = new CopyOnWriteArrayList<>(); static { List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); for (GarbageCollectorMXBean gcBean : gcBeans) { NotificationEmitter emitter = (NotificationEmitter) gcBean; emitter.addNotificationListener((notification, o) -> { + long uptimeInMillis = runtimeMXBean.getUptime(); while (! gcInfos.isEmpty()) { - if (System.currentTimeMillis() - gcInfos.get(0).getEndTime() > 5000) { + if (uptimeInMillis - gcInfos.get(0).getEndTime() > 5000) { gcSum -= gcInfos.remove(0).getDuration(); } else { break; @@ -104,8 +107,8 @@ abstract class MemoryOverflowModel extends AbstractModel implements AutoCloseabl GcInfo gcInfo = gcNotificationInfo.getGcInfo(); gcInfos.add(gcInfo); gcSum += gcInfo.getDuration(); - System.out.println("gcSum: " + gcSum); if (gcSum > 1000 || gcInfos.size() > 4) { + System.out.println("high gc load: sum=" + gcSum + " count=" + gcInfos.size()); highGcLoad = true; lastGcUpdate = System.currentTimeMillis(); } else if (System.currentTimeMillis() - lastGcUpdate > 10000) {