benchmark: rocksdb concurrent reads/writes

apps/omg_db/benchmark/read_utxos.exs

@@ -0,0 +1,73 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Usage: mix run apps/omg_db/benchmark/read_utxos.exs
+#
+
+defmodule OMG.DB.Benchmark.Helper do
+  @moduledoc false
+
+  def populate_utxos(server_pid, num_utxos) do
+    1..num_utxos
+    |> Enum.map(fn _index ->
+      output_data = %{
+        creating_txhash: random_bytes(),
+        output: %{owner: random_bytes(), currency: random_bytes(), amount: 1_000_000_000, output_type: 1}
+      }
+
+      # Randomized `{blknum, txindex, oindex}`
+      # Assuming potential storage of 100,000 blocks, 7,000 transactions per block and all outputs used
+      {:put, :utxo, {{:rand.uniform(100_000) * 1000, :rand.uniform(7000), :rand.uniform(4) - 1}, output_data}}
+    end)
+    |> OMG.DB.multi_update(server_pid)
+  end
+
+  defp random_bytes() do
+    0..255 |> Enum.shuffle() |> :erlang.list_to_binary()
+  end
+end
+
+alias OMG.DB.Benchmark.Helper
+
+:ok = Logger.configure(level: :warn)
+{:ok, _} = Application.ensure_all_started(:briefly)
+
+Benchee.run(
+  %{
+    "OMG.DB.utxos" => fn {_dir, server_pid} ->
+      {:ok, _utxos} = OMG.DB.utxos(server_pid)
+    end
+  },
+  inputs: %{
+    "1,000 utxos" => 1000,
+    "10,000 utxos" => 10_000,
+    "100,000 utxos" => 100_000
+  },
+  before_scenario: fn num_utxos ->
+    {:ok, dir} = Briefly.create(directory: true)
+    :ok = OMG.DB.RocksDB.Server.init_storage(dir)
+    {:ok, server_pid} = GenServer.start_link(OMG.DB.RocksDB.Server, [db_path: dir, name: :benchmark_db])
+
+    :ok = Helper.populate_utxos(server_pid, num_utxos)
+
+    {dir, server_pid}
+  end,
+  after_scenario: fn {dir, server_pid} ->
+    true = Process.exit(server_pid, :shutdown)
+    {:ok, _} = File.rm_rf(dir)
+  end,
+  time: 10,
+  memory_time: 2
+)

apps/omg_db/benchmark/read_utxos.txt

@@ -0,0 +1,51 @@
+Operating System: macOS
+CPU Information: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
+Number of Available Cores: 12
+Available memory: 16 GB
+Elixir 1.10.2
+Erlang 22.3
+
+Benchmark suite executing with the following configuration:
+warmup: 2 s
+time: 10 s
+memory time: 2 s
+parallel: 1
+inputs: 1,000 utxos, 10,000 utxos, 100,000 utxos
+Estimated total run time: 42 s
+
+Benchmarking OMG.DB.utxos with input 1,000 utxos...
+Benchmarking OMG.DB.utxos with input 10,000 utxos...
+Benchmarking OMG.DB.utxos with input 100,000 utxos...
+
+##### With input 1,000 utxos #####
+Name                   ips        average  deviation         median         99th %
+OMG.DB.utxos        168.95        5.92 ms    ±16.03%        5.72 ms        9.61 ms
+
+Memory usage statistics:
+
+Name            Memory usage
+OMG.DB.utxos           168 B
+
+**All measurements for memory usage were the same**
+
+##### With input 10,000 utxos #####
+Name                   ips        average  deviation         median         99th %
+OMG.DB.utxos         15.86       63.06 ms     ±4.03%       63.01 ms       69.31 ms
+
+Memory usage statistics:
+
+Name            Memory usage
+OMG.DB.utxos           168 B
+
+**All measurements for memory usage were the same**
+
+##### With input 100,000 utxos #####
+Name                   ips        average  deviation         median         99th %
+OMG.DB.utxos          1.33      753.83 ms     ±7.29%      759.87 ms      836.05 ms
+
+Memory usage statistics:
+
+Name            Memory usage
+OMG.DB.utxos           168 B
+
+**All measurements for memory usage were the same**

apps/omg_db/benchmark/read_utxos_breakdown.exs

@@ -0,0 +1,164 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Usage: mix run apps/omg_db/benchmark/utxo_get_breakdown.exs
+#
+
+defmodule OMG.DB.Benchmark.Helper do
+  @moduledoc false
+
+  @doc """
+  Initialize a new rocksdb database in a temp directory
+  and populate it with the given number of utxos.
+  """
+  def setup_db_with_data(num_utxos) do
+    {:ok, dir} = Briefly.create(directory: true)
+    :ok = OMG.DB.RocksDB.Server.init_storage(dir)
+
+    setup = [{:create_if_missing, false}, {:prefix_extractor, {:fixed_prefix_transform, 5}}]
+    db_path = String.to_charlist(dir)
+    {:ok, db_ref} = :rocksdb.open(db_path, setup)
+
+    :ok = populate_utxos(db_ref, num_utxos)
+
+    # Closing the instance for data population, each test will start again with their own config
+    :ok = :rocksdb.close(db_ref)
+    {:ok, dir}
+  end
+
+  def populate_utxos_every(interval_ms, :infinity, num_utxos, db_ref) do
+    populate_utxos_every(interval_ms, -1, num_utxos, db_ref)
+  end
+
+  def populate_utxos_every(_, 0, _, _), do: :ok
+
+  def populate_utxos_every(interval_ms, times, num_utxos, db_ref) do
+    :ok = populate_utxos(db_ref, num_utxos)
+    :ok = Process.sleep(interval_ms)
+
+    populate_utxos_every(interval_ms, times - 1, num_utxos, db_ref)
+  end
+
+  # Copied from OMG.DB.RocksDB.Core.search/4
+  def search(_reference, _iterator, {:error, :invalid_iterator}, acc), do: acc
+  def search(reference, iterator, {:ok, _key, value}, acc), do: do_search(reference, iterator, [value | acc])
+
+  def do_search(reference, iterator, acc) do
+    case :rocksdb.iterator_move(iterator, :next) do
+      {:error, :invalid_iterator} ->
+        # we've reached the end
+        :rocksdb.iterator_close(iterator)
+        acc
+
+      {:ok, _key, value} ->
+        do_search(reference, iterator, [value | acc])
+    end
+  end
+
+  defp random_bytes() do
+    0..255 |> Enum.shuffle() |> :erlang.list_to_binary()
+  end
+
+  def populate_utxos(db_ref, num_utxos) do
+    1..num_utxos
+    |> Enum.map(fn _index ->
+      output_data = %{
+        creating_txhash: random_bytes(),
+        output: %{owner: random_bytes(), currency: random_bytes(), amount: 1_000_000_000, output_type: 1}
+      }
+
+      # Randomized `{blknum, txindex, oindex}`
+      # Assuming potential load of 100,000 blocks, 7,000 transactions per block and all outputs used
+      {:put, :utxo, {{:rand.uniform(100_000) * 1000, :rand.uniform(7000), :rand.uniform(4) - 1}, output_data}}
+    end)
+    |> multi_update(db_ref)
+  end
+
+  # Stuff from OMG.DB.RocksDB.Core
+  defp multi_update(db_updates, db_ref) do
+    db_updates
+    |> Enum.flat_map(&parse_multi_update/1)
+    |> write(db_ref)
+  end
+
+  # Stuff from OMG.DB.RocksDB.Core
+  defp parse_multi_update({:put, type, item}), do: [{:put, key_for_item(type, item), encode_value(type, item)}]
+  defp parse_multi_update({:delete, type, item}), do: [{:delete, key(type, item)}]
+  defp key_for_item(:utxo, {position, _utxo}), do: key(:utxo, position)
+  defp key(:utxo, position), do: "utxoi" <> :erlang.term_to_binary(position)
+  defp encode_value(_type, value), do: :erlang.term_to_binary(value)
+
+  # Stuff from OMG.DB.RocksDB.Core
+  defp write(operations, db_ref) do
+    :rocksdb.write(db_ref, operations, [])
+  end
+end
+
+alias OMG.DB.Benchmark.Helper
+
+# Start
+{:ok, _} = Application.ensure_all_started(:briefly)
+
+# Benchmark run
+
+Benchee.run(
+  %{
+    "iterate" => fn {_dir, db_ref, _load_generator_pid} ->
+      {:ok, iterator} = :rocksdb.iterator(db_ref, prefix_same_as_start: true)
+      move_iterator = :rocksdb.iterator_move(iterator, {:seek, "utxoi"})
+      _raw_utxos = Helper.search(db_ref, iterator, move_iterator, [])
+    end,
+    "iterate_decode" => fn {_dir, db_ref, _load_generator_pid} ->
+      {:ok, iterator} = :rocksdb.iterator(db_ref, prefix_same_as_start: true)
+      move_iterator = :rocksdb.iterator_move(iterator, {:seek, "utxoi"})
+      raw_utxos = Helper.search(db_ref, iterator, move_iterator, [])
+
+      {:ok, _utxos} = OMG.DB.RocksDB.Core.decode_values(raw_utxos)
+    end,
+    "iterate_reverse_decode" => fn {_dir, db_ref, _load_generator_pid} ->
+      {:ok, iterator} = :rocksdb.iterator(db_ref, prefix_same_as_start: true)
+      move_iterator = :rocksdb.iterator_move(iterator, {:seek, "utxoi"})
+      raw_utxos = Enum.reverse(Helper.search(db_ref, iterator, move_iterator, []))
+
+      {:ok, _utxos} = OMG.DB.RocksDB.Core.decode_values(raw_utxos)
+    end
+  },
+  inputs: %{
+    "1,000 utxos" => 1000,
+    "10,000 utxos" => 10_000,
+    "100,000 utxos" => 100_000
+  },
+  before_scenario: fn num_utxos ->
+    {:ok, dir} = Helper.setup_db_with_data(num_utxos)
+
+    setup = [{:create_if_missing, false}, {:prefix_extractor, {:fixed_prefix_transform, 5}}]
+    {:ok, db_ref} = :rocksdb.open(String.to_charlist(dir), setup)
+
+    {:ok, load_generator_pid} =
+      Task.start_link(fn ->
+        # Every 100ms, populate outputs for 20 transactions with 4 outputs each
+        Helper.populate_utxos_every(100, :infinity, 20 * 4, db_ref)
+      end)
+
+    {dir, db_ref, load_generator_pid}
+  end,
+  after_scenario: fn {dir, db_ref, load_generator_pid} ->
+    true = Process.exit(load_generator_pid, :shutdown)
+    :ok = :rocksdb.close(db_ref)
+    {:ok, _} = File.rm_rf(dir)
+  end,
+  time: 10,
+  memory_time: 2
+)

apps/omg_db/benchmark/read_utxos_breakdown.txt

@@ -0,0 +1,93 @@
+Operating System: macOS
+CPU Information: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
+Number of Available Cores: 12
+Available memory: 16 GB
+Elixir 1.10.2
+Erlang 22.3
+
+Benchmark suite executing with the following configuration:
+warmup: 2 s
+time: 10 s
+memory time: 2 s
+parallel: 1
+inputs: 1,000 utxos, 10,000 utxos, 100,000 utxos
+Estimated total run time: 2.10 min
+
+Benchmarking iterate with input 1,000 utxos...
+Benchmarking iterate with input 10,000 utxos...
+Benchmarking iterate with input 100,000 utxos...
+Benchmarking iterate_decode with input 1,000 utxos...
+Benchmarking iterate_decode with input 10,000 utxos...
+Benchmarking iterate_decode with input 100,000 utxos...
+Benchmarking iterate_reverse_decode with input 1,000 utxos...
+Benchmarking iterate_reverse_decode with input 10,000 utxos...
+Benchmarking iterate_reverse_decode with input 100,000 utxos...
+
+##### With input 1,000 utxos #####
+Name                             ips        average  deviation         median         99th %
+iterate                        55.21       18.11 ms    ±39.53%       16.38 ms       34.71 ms
+iterate_reverse_decode         35.98       27.80 ms    ±39.28%       25.75 ms       55.54 ms
+iterate_decode                 35.85       27.89 ms    ±41.73%       24.98 ms       56.84 ms
+
+Comparison:
+iterate                        55.21
+iterate_reverse_decode         35.98 - 1.53x slower +9.68 ms
+iterate_decode                 35.85 - 1.54x slower +9.78 ms
+
+Memory usage statistics:
+
+Name                           average  deviation         median         99th %
+iterate                        1.13 MB     ±4.05%        1.13 MB        1.21 MB
+iterate_reverse_decode         4.33 MB     ±3.17%        4.34 MB        4.54 MB
+iterate_decode                 4.23 MB     ±4.02%        4.24 MB        4.51 MB
+
+Comparison:
+iterate                        1.13 MB
+iterate_reverse_decode         4.33 MB - 3.83x memory usage +3.20 MB
+iterate_decode                 4.23 MB - 3.74x memory usage +3.10 MB
+
+##### With input 10,000 utxos #####
+Name                             ips        average  deviation         median         99th %
+iterate                        15.71       63.66 ms    ±12.08%       62.19 ms       82.58 ms
+iterate_reverse_decode         10.50       95.22 ms    ±12.52%       94.53 ms      124.67 ms
+iterate_decode                 10.50       95.23 ms    ±14.59%       94.04 ms      139.46 ms
+
+Comparison:
+iterate                        15.71
+iterate_reverse_decode         10.50 - 1.50x slower +31.56 ms
+iterate_decode                 10.50 - 1.50x slower +31.57 ms
+
+Memory usage statistics:
+
+Name                           average  deviation         median         99th %
+iterate                        2.31 MB     ±2.04%        2.31 MB        2.39 MB
+iterate_reverse_decode         8.90 MB     ±1.66%        8.90 MB        9.12 MB
+iterate_decode                 8.64 MB     ±2.12%        8.63 MB        8.94 MB
+
+Comparison:
+iterate                        2.31 MB
+iterate_reverse_decode         8.90 MB - 3.85x memory usage +6.59 MB
+iterate_decode                 8.64 MB - 3.74x memory usage +6.33 MB
+
+##### With input 100,000 utxos #####
+Name                             ips        average  deviation         median         99th %
+iterate                         1.94      515.39 ms     ±2.93%      517.44 ms      544.25 ms
+iterate_reverse_decode          1.25      803.14 ms     ±5.01%      807.43 ms      872.77 ms
+iterate_decode                  1.19      838.78 ms     ±4.43%      836.77 ms      906.42 ms
+
+Comparison:
+iterate                         1.94
+iterate_reverse_decode          1.25 - 1.56x slower +287.75 ms
+iterate_decode                  1.19 - 1.63x slower +323.39 ms
+
+Memory usage statistics:
+
+Name                           average  deviation         median         99th %
+iterate                       14.00 MB     ±0.42%       14.00 MB       14.07 MB
+iterate_reverse_decode        54.40 MB     ±0.37%       54.40 MB       54.54 MB
+iterate_decode                52.64 MB     ±0.37%       52.64 MB       52.77 MB
+
+Comparison:
+iterate                       14.00 MB
+iterate_reverse_decode        54.40 MB - 3.89x memory usage +40.40 MB
+iterate_decode                52.64 MB - 3.76x memory usage +38.64 MB