From 4508a0aa4a49fcb70795b5118a67facbb8f1d388 Mon Sep 17 00:00:00 2001
From: popcnt <ism3y@proton.me>
Date: Sun, 7 Jul 2024 23:27:27 +0800
Subject: [PATCH] chore: refine file cache management in genesis data import

FileCacheManager has been integrated into the genesis_utxo.rs and genesis_ord.rs files. In the process of importing data, this allows for a detailed control over the memory cache, potentially enhancing performance. The respective bytes read calculations have also been optimized.

TODO: read file ahead
---
 .../rooch/src/commands/statedb/commands/genesis_ord.rs   | 3 +--
 .../rooch/src/commands/statedb/commands/genesis_utxo.rs  | 9 +++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/crates/rooch/src/commands/statedb/commands/genesis_ord.rs b/crates/rooch/src/commands/statedb/commands/genesis_ord.rs
index 27c598d427..30b5da56dd 100644
--- a/crates/rooch/src/commands/statedb/commands/genesis_ord.rs
+++ b/crates/rooch/src/commands/statedb/commands/genesis_ord.rs
@@ -466,12 +466,12 @@ fn produce_ord_updates(tx: SyncSender<BatchUpdatesOrd>, input: PathBuf, batch_si
 
         for line in reader.by_ref().lines().take(batch_size) {
             let line = line.unwrap();
+            bytes_read += line.len() as u64 + 1; // Add line.len() + 1, assuming that the line terminator is '\n'
 
             if is_title_line {
                 is_title_line = false;
                 if line.starts_with("# export at") {
                     // skip block height info
-                    bytes_read += line.len() as u64 + 1; // Add line.len() + 1, assuming that the line terminator is '\n'
                     continue;
                 }
             }
@@ -488,7 +488,6 @@ fn produce_ord_updates(tx: SyncSender<BatchUpdatesOrd>, input: PathBuf, batch_si
             let (key2, state2) = gen_inscription_ids_update(index, inscription_id);
             updates.inscription_ids_updates.put(key2, state2);
             index += 1;
-            bytes_read += line.len() as u64 + 1;
         }
         let _ = file_cache_mgr.drop_cache_range(cache_drop_offset, bytes_read);
         cache_drop_offset += bytes_read;
diff --git a/crates/rooch/src/commands/statedb/commands/genesis_utxo.rs b/crates/rooch/src/commands/statedb/commands/genesis_utxo.rs
index 51cd9a8e48..8f64a8a13f 100644
--- a/crates/rooch/src/commands/statedb/commands/genesis_utxo.rs
+++ b/crates/rooch/src/commands/statedb/commands/genesis_utxo.rs
@@ -22,6 +22,7 @@ use moveos_types::moveos_std::simple_multimap::{Element, SimpleMultiMap};
 use moveos_types::startup_info::StartupInfo;
 use moveos_types::state::{KeyState, MoveState, State};
 use redb::{Database, ReadOnlyTable};
+use rooch_common::fs::file_cache::FileCacheManager;
 use rooch_config::{RoochOpt, R_OPT_NET_HELP};
 use rooch_db::RoochDB;
 use rooch_types::address::BitcoinAddress;
@@ -399,6 +400,9 @@ pub fn produce_utxo_updates(
     batch_size: usize,
     utxo_ord_map_db: Option<Arc<Database>>,
 ) {
+    let file_cache_mgr = FileCacheManager::new(input.clone()).unwrap();
+    let mut cache_drop_offset: u64 = 0;
+
     let mut csv_reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(input).unwrap());
     let mut is_title_line = true;
     let mut address_mapping_checker = HashMap::new();
@@ -410,12 +414,15 @@ pub fn produce_utxo_updates(
         }
     };
     loop {
+        let mut bytes_read = 0;
+
         let mut updates = BatchUpdates {
             utxo_updates: UpdateSet::new(),
             rooch_to_bitcoin_mapping_updates: UpdateSet::new(),
         };
         for line in csv_reader.by_ref().lines().take(batch_size) {
             let line = line.unwrap();
+            bytes_read += line.len() as u64 + 1; // Add line.len() + 1, assuming that the line terminator is '\n'
 
             if is_title_line {
                 is_title_line = false;
@@ -447,6 +454,8 @@ pub fn produce_utxo_updates(
                 }
             }
         }
+        let _ = file_cache_mgr.drop_cache_range(cache_drop_offset, bytes_read);
+        cache_drop_offset += bytes_read;
         if updates.utxo_updates.is_empty() {
             break;
         }