[H2] Fixed RadixTreeMap so it can be properly rewritten during auto

compact. Fixes #231

sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVBTreeMap.java

@@ -14,6 +14,7 @@
 
 package org.h2.mvstore;
 
+import java.util.Set;
 import java.util.Spliterator;
 import java.util.Spliterators;
 import java.util.stream.Stream;
@@ -247,6 +248,92 @@ public Stream<K> keyStream()
         return StreamSupport.stream(it, false);
     }
 
+
+    /*
+     * Need to override this so pages and correctly rewritten on autoCompact
+     */
+    @Override
+    boolean rewrite(Set<Integer> set) {
+        // read from old version, to avoid concurrent reads
+        long previousVersion = store.getCurrentVersion() - 1;
+        if (previousVersion < getCreateVersion()) {
+            // a new map
+            return true;
+        }
+        MVMap<K, V> readMap;
+        try {
+            readMap = openVersion(previousVersion);
+        } catch (IllegalArgumentException e) {
+            // unknown version: ok
+            // TODO should not rely on exception handling
+            return true;
+        }
+        try {
+            synchronized (this)
+            {
+                beforeWrite();
+                long v = writeVersion;
+                var root = rewritePage(readMap.root, set, v);
+                if (root != null)
+                {
+                    newRoot(root);
+                    //System.out.println(getName() + ": Rewriting " + set.size() + " chunks: " + set);
+                }
+                return true;
+            }
+        } catch (IllegalStateException e) {
+            // TODO should not rely on exception handling
+            if (DataUtils.getErrorCode(e.getMessage()) == DataUtils.ERROR_CHUNK_NOT_FOUND) {
+                // ignore
+                return false;
+            }
+            throw e;
+        }
+    }
+
+    /*
+     * May rewrite the page if the page or one of its children is in one of the listed chunks.
+     * If a new page is created, it is returned, otherwise null is returned
+     * */
+    private Page rewritePage(Page p, Set<Integer> set, long version) {
+        long pos = p.getPos();
+        int chunkId = DataUtils.getPageChunkId(pos);
+
+        if (p.isLeaf()) {
+            return set.contains(chunkId) ? p.copy(version) : null;
+        }
+        else
+        {
+            Page newPage = set.contains(chunkId) ? p.copy(version) : null;
+
+            for (int i = 0; i < getChildPageCount(p); i++) {
+                // skip child page before even loading it if not in one of the chunks
+                long childPos = p.getChildPagePos(i);
+                if (childPos != 0)
+                {
+                    if (DataUtils.getPageType(childPos) == DataUtils.PAGE_TYPE_LEAF) {
+                        chunkId = DataUtils.getPageChunkId(childPos);
+                        if (!set.contains(chunkId))
+                            continue;
+                    }
+
+                    // may rewrite child page
+                    var newChild = rewritePage(p.getChildPage(i), set, version);
+
+                    // if child was rewritten, we also need to write this one
+                    // and all of the parents will be rewritten recursively
+                    if (newChild != null)
+                    {
+                        if (newPage == null)
+                            newPage = p.copy(version);
+                        newPage.setChild(i, newChild);
+                    }
+                }
+            }
+            return newPage;
+        }
+    }
+
 
     /**
      * A builder for this class.

sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVRadixTreeMap.java

@@ -967,6 +967,92 @@ protected void printValues(Page p, StringBuilder buf)
         }
     }
 
+
+    /*
+     * Need to override this so pages and correctly rewritten on autoCompact
+     */
+    @Override
+    boolean rewrite(Set<Integer> set) {
+        // read from old version, to avoid concurrent reads
+        long previousVersion = store.getCurrentVersion() - 1;
+        if (previousVersion < getCreateVersion()) {
+            // a new map
+            return true;
+        }
+        MVMap<K, V> readMap;
+        try {
+            readMap = openVersion(previousVersion);
+        } catch (IllegalArgumentException e) {
+            // unknown version: ok
+            // TODO should not rely on exception handling
+            return true;
+        }
+        try {
+            synchronized (this)
+            {
+                beforeWrite();
+                long v = writeVersion;
+                var root = rewritePage(readMap.root, set, v);
+                if (root != null)
+                {
+                    newRoot(root);
+                    //System.out.println(getName() + ": Rewriting " + set.size() + " chunks: " + set);
+                }
+                return true;
+            }
+        } catch (IllegalStateException e) {
+            // TODO should not rely on exception handling
+            if (DataUtils.getErrorCode(e.getMessage()) == DataUtils.ERROR_CHUNK_NOT_FOUND) {
+                // ignore
+                return false;
+            }
+            throw e;
+        }
+    }
+
+    /*
+     * May rewrite the page if the page or one of its children is in one of the listed chunks.
+     * If a new page is created, it is returned, otherwise null is returned
+     * */
+    private Page rewritePage(Page p, Set<Integer> set, long version) {
+        long pos = p.getPos();
+        int chunkId = DataUtils.getPageChunkId(pos);
+
+        if (p.isLeaf()) {
+            return set.contains(chunkId) ? p.copy(version) : null;
+        }
+        else
+        {
+            Page newPage = set.contains(chunkId) ? p.copy(version) : null;
+
+            for (int i = 0; i < getChildPageCount(p); i++) {
+                // skip child page before even loading it if not in one of the chunks
+                long childPos = p.getChildPagePos(i);
+                if (childPos != 0)
+                {
+                    if (DataUtils.getPageType(childPos) == DataUtils.PAGE_TYPE_LEAF) {
+                        chunkId = DataUtils.getPageChunkId(childPos);
+                        if (!set.contains(chunkId))
+                            continue;
+                    }
+
+                    // may rewrite child page
+                    var newChild = rewritePage(p.getChildPage(i), set, version);
+
+                    // if child was rewritten, we also need to write this one
+                    // and all of the parents will be rewritten recursively
+                    if (newChild != null)
+                    {
+                        if (newPage == null)
+                            newPage = p.copy(version);
+                        newPage.setChild(i, newChild);
+                    }
+                }
+            }
+            return newPage;
+        }
+    }
+
 
     /**
      * A builder for this class.

sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/PrintDbInfo.java

@@ -0,0 +1,79 @@
+/***************************** BEGIN LICENSE BLOCK ***************************
+
+The contents of this file are subject to the Mozilla Public License, v. 2.0.
+If a copy of the MPL was not distributed with this file, You can obtain one
+at http://mozilla.org/MPL/2.0/.
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+for the specific language governing rights and limitations under the License.
+ 
+Copyright (C) 2023 Sensia Software LLC. All Rights Reserved.
+ 
+******************************* END LICENSE BLOCK ***************************/
+
+package org.sensorhub.impl.datastore.h2;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+import org.h2.mvstore.DataUtils;
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.Page;
+
+
+public class PrintDbInfo
+{
+
+    public static void main(String[] args) throws Exception
+    {
+        var db = new MVObsSystemDatabase();
+        var config = new MVObsSystemDatabaseConfig();
+        config.databaseNum = 0;
+        //config.storagePath = "/home/alex/Projects/Workspace_OSH_V2/tests/db_growing_bug/piaware_austin.dat";
+        config.storagePath = "/home/alex/Projects/Workspace_OSH_V2/osh-addons-grx/sensorhub-test-grx/all_obs_h2_t18.dat";
+        db.init(config);
+        db.start();
+
+        scanPages(db.foiStore.featuresIndex);
+        scanPages(db.foiStore.idsIndex);
+        scanPages(db.foiStore.uidsIndex);
+        //scanPages(db.foiStore.fullTextIndex.radixTreeMap);
+        //scanPages(db.foiStore.spatialIndex.rTreeMap);
+
+        //Thread.sleep(100000);
+        db.stop();
+    }
+
+
+    static void scanPages(MVMap<?,?> map)
+    {
+        var chunkIds = new LinkedHashSet<Integer>();
+        scanPage(map.getRoot(), chunkIds);
+
+        System.out.println("**************************");
+        System.out.println("Map " + map.getName() + ", size=" + map.size());
+        System.out.println(chunkIds);
+        System.out.println();
+    }
+
+
+    static void scanPage(Page page, Set<Integer> chunkIds)
+    {
+        var pos = page.getPos();
+        var chunkId = DataUtils.getPageChunkId(pos);
+        chunkIds.add(chunkId);
+
+        if (!page.isLeaf())
+        {
+            for (int i = 0; i < page.getRawChildPageCount(); i++)
+            {
+                var childPos = page.getChildPagePos(i);
+                if (childPos != 0)
+                {
+                    var childPage = page.getChildPage(i);
+                    scanPage(childPage, chunkIds);
+                }
+            }
+        }
+    }
+}