Block file and block manager tests

test/asami/durable/block/test_blockfile.clj

@@ -0,0 +1,125 @@
+(ns ^{:doc "Tests for the BlockFile implementation"
+      :author "Paula Gearon"}
+    asami.durable.block.test-blockfile
+  (:require [clojure.test :refer :all]
+            [asami.durable.block.block-api :refer :all]
+            [asami.durable.block.block-file :refer :all]
+            [asami.durable.block.file.voodoo :refer [windows?]]))
+
+(def test-block-size 256)
+
+(defn cleanup
+  []
+  (when voodoo/windows?
+    (System/gc)
+    (System/runFinalization)))
+
+(defn exec-bf
+  [filename f]
+  (let [filename (util/temp-file filename)
+        {:keys [block-file file]} (open-block-file filename test-block-size)]
+    (try
+      (f file block-file)
+      (finally
+        (clear! block-file)
+        (.close file)
+        (cleanup)))))
+
+(defmacro with-block-file
+  "Executes the body in a context of an unmanaged block file"
+  [filename body]
+  `(exec-bf ~filename (fn [bf af] ~@body)))
+
+(def utf8 (Charset/forName "UTF-8"))
+
+(defn put-string! [b s]
+  (let [^bytes bytes (.getBytes s utf8)]
+    (put-byte! b 0 (count bytes))
+    (put-bytes! b 1 bytes)))
+
+(defn get-string [b]
+  (let [l (get-byte b 0)
+        d (get-bytes b 1 l)]
+    (String. d utf8)))
+
+(deftest test-allocate
+  (let [filename (util/temp-file "ualloc")
+        {:keys [block-file file]} (open-block-file filename test-block-size)]
+    (set-nr-blocks! block-file 1)
+    (try
+      (let [blk (block-for block-file 0)]
+        (is (not (nil? blk))))
+      (finally
+        (clear! block-file)
+        (.close file)
+        (cleanup)))))
+
+(deftest test-write
+  (let [file-str "bftest"
+        filename (util/temp-file file-str)
+        {:keys [block-file file]} (open-block-file filename test-block-size)
+        bf (set-nr-blocks! block-file 4)]
+    (try
+      (let [b (block-for bf 0)
+            _ (put-string! b str0)
+            b (block-for bf 3)
+            _ (put-string! b str3)
+            b (block-for bf 2)
+            _ (put-string! b str2)
+            b (block-for bf 1)
+            _ (put-string! b str1)]
+
+        (is (= str2 (get-string (block-for bf 2))))
+        (is (= str0 (get-string (block-for bf 0))))
+        (is (= str1 (get-string (block-for bf 1))))
+        (is (= str3 (get-string (block-for bf 3)))))
+      (finally
+        (clear! bf)
+        (.close file)
+        (cleanup)))
+
+    ;; close all, and start again
+    (unmap bf)
+    (.close file)
+    (cleanup)
+    (let [{:keys [block-file file]} (open-block-file filename test-block-size)]
+
+      ;; did it persist
+
+      (is (= 4 (get-nr-blocks block-file)))
+
+      (is (= str2 (get-string (block-for block-file 2))))
+      (is (= str0 (get-string (block-for block-file 0))))
+      (is (= str1 (get-string (block-for block-file 1))))
+      (is (= str3 (get-string (block-for block-file 3)))))))
+
+(deftest test-performance
+  (let [file-str "perftest"
+        filename (util/temp-file file-str)
+        {:keys [block-file file]} (open-block-file filename test-block-size)
+        _ (clear! block-file)
+        nr-blocks 100000
+        bf (set-nr-blocks! block-file nr-blocks)]
+
+    (try
+      (doseq [i (range nr-blocks)]
+        (let [b (block-for bf i)]
+          (put-int! b 0 (+ i 5))))
+
+      (doseq [i (range nr-blocks)]
+        (let [b (block-for bf i)]
+          (is (= (+ i 5) (get-int b 0)))))
+
+      (doseq [pass (range 10)]
+        (doseq [i (range nr-blocks)]
+          (let [b (block-for bf i)]
+            (put-int! b 0 (bit-xor i pass))))
+        (doseq [i (range nr-blocks)]
+          (let [b (block-for bf i)]
+            (is (= (bit-xor i pass) (get-int b 0))))))
+
+      (finally
+        (clear! bf)
+        (.close file)
+        (cleanup)))))
+

test/asami/durable/block/test_blockmanager.cljc

@@ -0,0 +1,154 @@
+(ns ^{:doc "Tests for the ManagedBlockFile implementation"
+      :author "Paula Gearon"}
+    asami.durable.block.test-blockmanager
+  (:require [clojure.test :refer :all]
+            [asami.durable.block.block-api :refer :all]
+            [asami.durable.block.block-file :refer :all]
+            [asami.durable.block.file.voodoo :refer [windows?]]))
+
+(def test-block-size 256)
+
+(defn cleanup
+  []
+  (when voodoo/windows?
+    (System/gc)
+    (System/runFinalization)))
+
+(defn get-filename
+  "Returns the resource for creating a manager.
+  For Java, this is a java.io.File. On JS this is a string."
+  [s]
+  #?(:clj (let [f (util/temp-file s)]
+            (.delete d))
+     :cljs s))
+
+(defn create-block-manager
+  "Central BlockManager construction. On the JVM this is ManagedBlockFile.
+  Takes a string argument (for construction), and block size."
+  [s sz]
+  (let [f (get-filename s)]
+    [f
+     #?(:clj (create-managed-block-file f sz))]))
+
+(defn recreate-block-manager
+  "Central BlockManager construction. On the JVM this is ManagedBlockFile.
+  Takes an argument for creating the manager (e.g. File, or string), and a size."
+  [f sz]
+  #?(:clj (create-managed-block-file f sz)))
+
+(defn remove
+  "Remove block manager resources."
+  [f]
+  #?(:clj (.delete f)))
+
+(deftest test-allocate
+  (let [[filename mbf] (create-block-manager "alloc" test-block-size)]
+    (try
+      (let [blk (allocate-block mbf)]
+        (is (not (nil? blk))))
+      (finally
+        (close mbf)
+        (cleanup)
+        (remove filename)))))
+
+(deftest test-write
+  (let [[filename mbf] (create-block-manager "mbwrite" test-block-size)
+        ids (volatile! nil)]
+    (try
+      (let [b0 (allocate-block mbf)
+            id0 (get-id b0)
+            _ (put-string! b0 str0)
+            b3 (allocate-block mbf)
+            id3 (get-id b3)
+            _ (put-string! b3 str3)
+            b2 (allocate-block mbf)
+            id2 (get-id b2)
+            _ (put-string! b2 str2)
+            b1 (allocate-block mbf)
+            id1 (get-id b1)
+            _ (put-string! b str1)]
+        (vreset! ids [id0 id1 id2 id3])
+
+        (is (= str2 (get-string (get-block mbf id2))))
+        (is (= str0 (get-string (get-block mbf id0))))
+        (is (= str1 (get-string (get-block mbf id1))))
+        (is (= str3 (get-string (get-block mbf id3)))))
+      (finally
+        (close mbf)))
+
+    ;; close all, and start again
+    (cleanup)
+    (try
+      (let [[id0 id1 id2 id3] @ids
+            mbf2 (recreate-block-manager filename test-block-size)]
+
+        #?(:clj
+           ;; did it persist?
+           (is (= 4 (get-nr-blocks (:block-file mbf2)))))
+
+        (is (= str2 (get-string (get-block mbf2 id2))))
+        (is (= str0 (get-string (get-block mbf2 id0))))
+        (is (= str1 (get-string (get-block mbf2 id1))))
+        (is (= str3 (get-string (get-block mbf2 id3)))))
+      (finally
+        (close mbf2)
+        (cleanup)
+        (remove filename)))))
+
+(deftest test-performance
+  (let [[filename mbf] (create-block-manager "perftest" Long/BYTES)
+        nr-blocks 100000]
+
+    ;; put numbers in the first 100,000 blocks
+    (try
+      (let [rand-mem (reduce (fn [m i]
+                               (let [n (long (rand nr-blocks))
+                                     b (alloc-block mbf)]
+                                 (put-long! b 0 n)
+                                 (write-block! mbf b)
+                                 (assoc m (get-id b) n)))
+                             {} (range nr-blocks))
+            ;; commit the first 100,000
+            mbf (commit! mbf)
+            ;; put numbers in the next 100,000 blocks
+            rand-mem2 (reduce (fn [m i]
+                                (let [n (long (rand nr-blocks))
+                                      b (alloc-block mbf)]
+                                  (put-long! b 0 n)
+                                  (write-block! mbf b)
+                                  (assoc m (get-id b) n)))
+                              {} (range nr-blocks (* 2 nr-blocks)))]
+
+        ;; check that all 200,000 numbers were stored as expected
+        (doseq [[i r] (concat rand-mem rand-mem2)]
+          (let [b (get-block mbf i)]
+            (is (= r (get-long b 0)))))
+
+        ;; rewind, to reuse the second 100,000 blocks
+        (let [mbf (rewind! mbf)
+              rand-mem3 (reduce (fn [m i]
+                                  (let [n (long (rand nr-blocks))
+                                        b (alloc-block mbf)]
+                                    (put-long! b 0 n)
+                                    (write-block! mbf b)
+                                    (assoc m (get-id b) n)))
+                                {} (range nr-blocks (* 2 nr-blocks)))]
+
+          ;; check that both first 100,000 is still right
+          ;; and the second 100,000 contains the new data
+          (doseq [[i r] (concat rand-mem rand-mem3)]
+            (let [b (get-block mbf i)]
+              (is (= r (get-long b 0)))))
+
+          ;; drop the second 100,000 again
+          (let [mbf (rewind! mbf)]
+            ;; truncate the data after the first 100,000
+            (close mbf)
+            #?(:clj
+               ;; file ManagedBlockFile, did the file get truncated to the first 100,000?
+               (when (instance? ManagedBlockFile mbf)
+                 (is (= (* nr-blocks Long/BYTES) (.length filename))))))))
+
+      (finally
+        (cleanup)
+        (remove filename)))))