Skip to content

Commit

Permalink
Add stats command
Browse files Browse the repository at this point in the history
  • Loading branch information
sharkAndshark committed Nov 7, 2023
1 parent 73b56e8 commit 29c1fba
Show file tree
Hide file tree
Showing 7 changed files with 326 additions and 4 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions mbtiles/src/bin/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ enum Commands {
/// MBTiles file to read from
file: PathBuf,
},
/// Gets tile statistics from MBTiels file
#[command(name = "stats")]
Stats { file: PathBuf },
/// Gets a single value from the MBTiles metadata table.
#[command(name = "meta-get")]
MetaGetValue {
Expand Down Expand Up @@ -114,6 +117,13 @@ async fn main_int() -> anyhow::Result<()> {
let mbt = Mbtiles::new(file.as_path())?;
mbt.validate(integrity_check, update_agg_tiles_hash).await?;
}
Commands::Stats { file } => {
let mbt = Mbtiles::new(file.as_path())?;
let mut conn = mbt.open_readonly().await?;

let statistics = mbt.statistics(&mut conn).await?;
println!("{statistics}");
}
}

Ok(())
Expand Down
216 changes: 212 additions & 4 deletions mbtiles/src/mbtiles.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

use std::collections::HashSet;
use std::ffi::OsStr;
use std::fmt::Display;
use std::path::Path;
use std::fmt::{Display, Formatter};
use std::path::{Path, PathBuf};
use std::str::FromStr;

#[cfg(feature = "cli")]
Expand Down Expand Up @@ -39,6 +39,60 @@ pub struct Metadata {
pub json: Option<JSONValue>,
}

#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct LevelDetail {
pub zoom: String,
pub count: u32,
pub smallest: f32,
pub largest: f32,
pub average: f32,
pub bounding_box: [f32; 4],
}

#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Statistics {
pub file_path: String,
pub file_size: f64,
pub schema: String,
pub page_size: Option<i32>,
pub level_details: Vec<LevelDetail>,
}

impl Display for Statistics {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
writeln!(f, "File: {}", self.file_path).unwrap();
writeln!(f, "FileSize: {:.2}MB", self.file_size).unwrap();
writeln!(f, "Schema: {}", self.schema).unwrap();
writeln!(f, "Page size: {} bytes", self.page_size.unwrap()).unwrap();
writeln!(
f,
"|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|",
"zoom", "count", "smallest", "largest", "average", "bbox"
)
.unwrap();

for l in &self.level_details {
let bbox_str = format!(
"{:.2}, {:.2}, {:.2}, {:.2}",
l.bounding_box[0], l.bounding_box[1], l.bounding_box[2], l.bounding_box[3]
);

writeln!(
f,
"|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|",
l.zoom,
l.count,
format!("{:.2}KB", l.smallest),
format!("{:.2}KB", l.largest),
format!("{:.2}KB", l.average),
bbox_str
)
.unwrap();
}
Ok(())
}
}

#[allow(clippy::trivially_copy_pass_by_ref)]
fn serialize_ti<S>(ti: &TileInfo, serializer: S) -> Result<S::Ok, S::Error>
where
Expand Down Expand Up @@ -107,7 +161,7 @@ pub struct Mbtiles {
}

impl Display for Mbtiles {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.filepath)
}
}
Expand Down Expand Up @@ -223,7 +277,114 @@ impl Mbtiles {
self.check_agg_tiles_hashes(&mut conn).await
}
}

pub async fn statistics<T>(&self, conn: &mut T) -> MbtResult<Statistics>
where
for<'e> &'e mut T: SqliteExecutor<'e>,
{
let file_size =
PathBuf::from(&self.filepath).metadata().unwrap().len() as f64 / 1024.0 / 1024.0;
let page_size_query = query!("PRAGMA page_size;");
let tile_infos_query = query!(
r#"SELECT
zoom_level AS zoom,
count( ) AS count,
min( length( tile_data ) ) / 1024.0 AS smallest,
max( length( tile_data ) ) / 1024.0 AS largest,
avg( length( tile_data ) ) / 1024.0 AS average,
min(tile_column) as min_tile_x,
min(tile_row) as min_tile_y,
max(tile_column) as max_tile_x,
max(tile_row) as max_tile_y
FROM tiles
GROUP BY zoom_level"#
);
let page_size = page_size_query.fetch_one(&mut *conn).await?.page_size;
let mb_type_string = match self.detect_type(&mut *conn).await? {
MbtType::Flat => "flat",
MbtType::FlatWithHash => "flat with hash",
MbtType::Normalized { .. } => "normalized",
};
let level_rows = tile_infos_query.fetch_all(&mut *conn).await?;
let mut level_details: Vec<LevelDetail> = level_rows
.into_iter()
.map(|r| {
let zoom = r.zoom.unwrap() as u8;
let count = r.count as u32;
let tile_length: f32 = 40075016.7 / (2_u32.pow(zoom as u32)) as f32;

let smallest = r.smallest.unwrap_or(0.0) as f32;
let largest = r.largest.unwrap_or(0.0) as f32;
let average = r.average.unwrap_or(0.0) as f32;

let min_tile_x = r.min_tile_x.unwrap();
let min_tile_y = r.min_tile_y.unwrap();
let max_tile_x = r.max_tile_x.unwrap();
let max_tile_y = r.max_tile_y.unwrap();

let minx = -20037508.34 + min_tile_x as f32 * tile_length;
let miny = -20037508.34 + min_tile_y as f32 * tile_length;
let maxx = -20037508.34 + (max_tile_x as f32 + 1.0) * tile_length;
let maxy = -20037508.34 + (max_tile_y as f32 + 1.0) * tile_length;

let bbox: [f32; 4] = [minx, miny, maxx, maxy];

LevelDetail {
zoom: format!("{zoom}"),
count,
smallest,
largest,
average,
bounding_box: bbox,
}
})
.collect();
let details_of_all = LevelDetail {
zoom: "all".to_string(),
count: level_details.iter().map(|l| l.count).sum(),
smallest: level_details
.iter()
.map(|l| l.smallest)
.reduce(f32::min)
.unwrap(),
largest: level_details
.iter()
.map(|l| l.largest)
.reduce(f32::max)
.unwrap(),
average: level_details.iter().map(|l| l.average).sum::<f32>()
/ level_details.len() as f32,
bounding_box: [
level_details
.iter()
.map(|l| l.bounding_box[0])
.reduce(f32::min)
.unwrap(),
level_details
.iter()
.map(|l| l.bounding_box[1])
.reduce(f32::min)
.unwrap(),
level_details
.iter()
.map(|l| l.bounding_box[2])
.reduce(f32::max)
.unwrap(),
level_details
.iter()
.map(|l| l.bounding_box[3])
.reduce(f32::max)
.unwrap(),
],
};
level_details.push(details_of_all);
Ok(Statistics {
file_path: self.filepath.clone(),
file_size,
schema: mb_type_string.to_owned(),
page_size,
level_details,
})
}
/// Get the aggregate tiles hash value from the metadata table
pub async fn get_agg_tiles_hash<T>(&self, conn: &mut T) -> MbtResult<Option<String>>
where
Expand Down Expand Up @@ -849,4 +1010,51 @@ mod tests {
assert!(matches!(result, Err(MbtError::AggHashMismatch(..))));
Ok(())
}

#[actix_rt::test]
async fn stat() -> MbtResult<()> {
let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?;
let res = mbt.statistics(&mut conn).await?;

assert_eq!(
res.file_path,
"../tests/fixtures/mbtiles/world_cities.mbtiles"
);
assert_eq!(res.file_size, 0.046875);
assert_eq!(res.schema, "flat");
assert_eq!(res.page_size, Some(4096));

assert_eq!(res.level_details.len(), 8);

assert_eq!(res.level_details[0].zoom, "0");
assert_eq!(res.level_details[0].count, 1);
assert_eq!(res.level_details[0].smallest, 1.0810547);
assert_eq!(res.level_details[0].largest, 1.0810547);
assert_eq!(res.level_details[0].average, 1.0810547);
assert_eq!(
res.level_details[0].bounding_box,
[-20037508.0, -20037508.0, 20037508.0, 20037508.0]
);

assert_eq!(res.level_details[6].zoom, "6");
assert_eq!(res.level_details[6].count, 72);
assert_eq!(res.level_details[6].smallest, 0.0625);
assert_eq!(res.level_details[6].largest, 0.09472656);
assert_eq!(res.level_details[6].average, 0.06669108);
assert_eq!(
res.level_details[6].bounding_box,
[-13775787.0, -5009377.0, 20037508.0, 8766410.0]
);

assert_eq!(res.level_details[7].zoom, "all");
assert_eq!(res.level_details[7].count, 196);
assert_eq!(res.level_details[7].smallest, 0.00);
assert_eq!(res.level_details[7].largest, 0.48);
assert_eq!(res.level_details[7].average, 0.13);
assert_eq!(
res.level_details[6].bounding_box,
[-20037508.00, -20037508.00, 20037508.00, 20037508.00]
);
Ok(())
}
}
1 change: 1 addition & 0 deletions tests/expected/mbtiles/help.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Usage: mbtiles <COMMAND>

Commands:
meta-all Prints all values in the metadata table in a free-style, unstable YAML format
stats Gets tile statistics from MBTiels file
meta-get Gets a single value from the MBTiles metadata table
meta-set Sets a single value in the MBTiles' file metadata table or deletes it if no value
copy Copy tiles from one mbtiles file to another
Expand Down
14 changes: 14 additions & 0 deletions tests/expected/mbtiles/stats.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
File: ./tests/fixtures/mbtiles/world_cities.mbtiles
FileSize: 0.05MB
Schema: flat
Page size: 4096 bytes
| zoom | count |smallest | largest | average | bbox |
| 0 | 1 | 1.08KB | 1.08KB | 1.08KB |-20037508.00, -20037508.00, 20037508.00, 20037508.00|
| 1 | 4 | 0.16KB | 0.63KB | 0.36KB |-20037508.00, -20037508.00, 20037508.00, 20037508.00|
| 2 | 7 | 0.13KB | 0.48KB | 0.23KB |-20037508.00, -10018754.00, 20037508.00, 10018754.00|
| 3 | 17 | 0.07KB | 0.24KB | 0.13KB |-15028131.00, -5009377.00, 20037508.00, 10018754.00|
| 4 | 38 | 0.06KB | 0.17KB | 0.08KB |-15028131.00, -5009377.00, 20037508.00, 10018754.00|
| 5 | 57 | 0.06KB | 0.10KB | 0.07KB |-13775787.00, -5009377.00, 20037508.00, 8766410.00|
| 6 | 72 | 0.06KB | 0.09KB | 0.07KB |-13775787.00, -5009377.00, 20037508.00, 8766410.00|
| all | 196 | 0.00KB | 0.48KB | 0.13KB |-20037508.00, -20037508.00, 20037508.00, 20037508.00|

1 change: 1 addition & 0 deletions tests/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ if [[ "$MBTILES_BIN" != "-" ]]; then
$MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt"
$MBTILES_BIN meta-all --help 2>&1 | tee "$TEST_OUT_DIR/meta-all_help.txt"
$MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt"
$MBTILES_BIN stats ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/stats.txt"
$MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt"
$MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt"
$MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"
Expand Down

0 comments on commit 29c1fba

Please sign in to comment.