Skip to content

Commit

Permalink
[Indexer] Optimize indexer query cache and pool core config (#2367)
Browse files Browse the repository at this point in the history
* optmize indexer cache size and mmap_size

* adjust indexer query mmap size
baichuan3 authored Aug 6, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 6035beb commit 6c5b599
Showing 2 changed files with 20 additions and 5 deletions.
2 changes: 1 addition & 1 deletion crates/rooch-indexer/src/indexer_reader.rs
Original file line number Diff line number Diff line change
@@ -108,7 +108,7 @@ pub struct IndexerReader {

impl IndexerReader {
pub fn new(db_path: PathBuf, registry: &Registry) -> Result<Self> {
let config = SqliteConnectionPoolConfig::default();
let config = SqliteConnectionPoolConfig::pool_config(true);
Self::new_with_config(db_path, config, registry)
}

23 changes: 19 additions & 4 deletions crates/rooch-indexer/src/lib.rs
Original file line number Diff line number Diff line change
@@ -202,6 +202,7 @@ pub struct SqliteConnectionPoolConfig {

impl SqliteConnectionPoolConfig {
const DEFAULT_POOL_SIZE: u32 = 16;
const DEFAULT_READER_POOL_SIZE: u32 = 32;
const DEFAULT_CONNECTION_TIMEOUT: u64 = 120; // second

fn connection_config(&self) -> SqliteConnectionConfig {
@@ -221,14 +222,17 @@ impl SqliteConnectionPoolConfig {
pub fn set_connection_timeout(&mut self, timeout: Duration) {
self.connection_timeout = timeout;
}
}

impl Default for SqliteConnectionPoolConfig {
fn default() -> Self {
pub fn pool_config(read_only: bool) -> Self {
let default_pool_size = if read_only {
Self::DEFAULT_READER_POOL_SIZE
} else {
Self::DEFAULT_POOL_SIZE
};
let db_pool_size = std::env::var("DB_POOL_SIZE")
.ok()
.and_then(|s| s.parse::<u32>().ok())
.unwrap_or(Self::DEFAULT_POOL_SIZE);
.unwrap_or(default_pool_size);
let conn_timeout_secs = std::env::var("DB_CONNECTION_TIMEOUT")
.ok()
.and_then(|s| s.parse::<u64>().ok())
@@ -241,6 +245,12 @@ impl Default for SqliteConnectionPoolConfig {
}
}

impl Default for SqliteConnectionPoolConfig {
fn default() -> Self {
Self::pool_config(false)
}
}

#[derive(Debug, Clone)]
struct SqliteConnectionConfig {
// SQLite does not support the statement_timeout parameter
@@ -274,11 +284,16 @@ impl diesel::r2d2::CustomizeConnection<SqliteConnection, diesel::r2d2::Error>
let mut pragma_builder = String::new();
if self.read_only {
pragma_builder.push_str("PRAGMA query_only = true;");
// The default cache_size value is -2000, which translates into a maximum of 2048000 bytes per cache.
// The cache_size in SQLite is primarily associated with the database connection, not the database file itself.
pragma_builder.push_str("PRAGMA cache_size = 65536000;"); // 64MB
}
// WAL mode has better write-concurrency. When synchronous is NORMAL it will fsync only in critical moments
if self.enable_wal {
pragma_builder.push_str("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;");
}
// The mmap_size in SQLite is primarily associated with the database file, not the connection.
pragma_builder.push_str("PRAGMA mmap_size = 268435456"); // 256MB
conn.batch_execute(&pragma_builder)
.map_err(diesel::r2d2::Error::QueryError)?;

0 comments on commit 6c5b599

Please sign in to comment.