From 31e55b03f889c1c8a10a7a755632b3b79f520d1c Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 1 Nov 2023 17:50:55 +0800 Subject: [PATCH] Add stats command --- ...962954cdd3d913e8a786599da8a3f9799ed4b.json | 20 +++ ...a8f175977ae491ea3d47069b20b1946c44ade.json | 68 ++++++++ mbtiles/src/bin/main.rs | 10 ++ mbtiles/src/mbtiles.rs | 165 +++++++++++++++++- tests/expected/mbtiles/help.txt | 1 + tests/expected/mbtiles/stats.txt | 13 ++ tests/test.sh | 1 + 7 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 mbtiles/.sqlx/query-208681caa7185b4014e7eda4120962954cdd3d913e8a786599da8a3f9799ed4b.json create mode 100644 mbtiles/.sqlx/query-e0e1a38ef162278d888b297ec85a8f175977ae491ea3d47069b20b1946c44ade.json create mode 100644 tests/expected/mbtiles/stats.txt diff --git a/mbtiles/.sqlx/query-208681caa7185b4014e7eda4120962954cdd3d913e8a786599da8a3f9799ed4b.json b/mbtiles/.sqlx/query-208681caa7185b4014e7eda4120962954cdd3d913e8a786599da8a3f9799ed4b.json new file mode 100644 index 000000000..3550f54e0 --- /dev/null +++ b/mbtiles/.sqlx/query-208681caa7185b4014e7eda4120962954cdd3d913e8a786599da8a3f9799ed4b.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "PRAGMA page_size;", + "describe": { + "columns": [ + { + "name": "page_size", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "208681caa7185b4014e7eda4120962954cdd3d913e8a786599da8a3f9799ed4b" +} diff --git a/mbtiles/.sqlx/query-e0e1a38ef162278d888b297ec85a8f175977ae491ea3d47069b20b1946c44ade.json b/mbtiles/.sqlx/query-e0e1a38ef162278d888b297ec85a8f175977ae491ea3d47069b20b1946c44ade.json new file mode 100644 index 000000000..d5c3f6e56 --- /dev/null +++ b/mbtiles/.sqlx/query-e0e1a38ef162278d888b297ec85a8f175977ae491ea3d47069b20b1946c44ade.json @@ -0,0 +1,68 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n zoom_level AS zoom,\n count( ) AS count,\n min( length( tile_data ) ) / 1024.0 AS smallest,\n max( length( tile_data ) ) / 1024.0 AS largest,\n avg( length( tile_data ) ) / 1024.0 AS average,\n min(tile_column) as min_tile_x,\n min(tile_row) as min_tile_y,\n max(tile_column) as max_tile_x,\n max(tile_row) as max_tile_y \n FROM tiles\n GROUP BY zoom_level", + "describe": { + "columns": [ + { + "name": "zoom", + "ordinal": 0, + "type_info": "Int64" + }, + { + "name": "count", + "ordinal": 1, + "type_info": "Int" + }, + { + "name": "smallest", + "ordinal": 2, + "type_info": "Float" + }, + { + "name": "largest", + "ordinal": 3, + "type_info": "Float" + }, + { + "name": "average", + "ordinal": 4, + "type_info": "Float" + }, + { + "name": "min_tile_x", + "ordinal": 5, + "type_info": "Int" + }, + { + "name": "min_tile_y", + "ordinal": 6, + "type_info": "Int" + }, + { + "name": "max_tile_x", + "ordinal": 7, + "type_info": "Int" + }, + { + "name": "max_tile_y", + "ordinal": 8, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + true, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "e0e1a38ef162278d888b297ec85a8f175977ae491ea3d47069b20b1946c44ade" +} diff --git a/mbtiles/src/bin/main.rs b/mbtiles/src/bin/main.rs index 9643e5f54..2073bbbf9 100644 --- a/mbtiles/src/bin/main.rs +++ b/mbtiles/src/bin/main.rs @@ -26,6 +26,9 @@ enum Commands { /// MBTiles file to read from file: PathBuf, }, + /// Gets tile statistics from MBTiels file + #[command(name = "stats")] + Stats { file: PathBuf }, /// Gets a single value from the MBTiles metadata table. #[command(name = "meta-get")] MetaGetValue { @@ -114,6 +117,13 @@ async fn main_int() -> anyhow::Result<()> { let mbt = Mbtiles::new(file.as_path())?; mbt.validate(integrity_check, update_agg_tiles_hash).await?; } + Commands::Stats { file } => { + let mbt = Mbtiles::new(file.as_path())?; + let mut conn = mbt.open_readonly().await?; + + let statistics = mbt.statistics(&mut conn).await?; + println!("{statistics}"); + } } Ok(()) diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index 2faf2b894..a224362d5 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -2,8 +2,8 @@ use std::collections::HashSet; use std::ffi::OsStr; -use std::fmt::Display; -use std::path::Path; +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; use std::str::FromStr; #[cfg(feature = "cli")] @@ -39,6 +39,60 @@ pub struct Metadata { pub json: Option, } +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct LevelDetail { + pub zoom: u8, + pub count: u32, + pub smallest: f32, + pub largest: f32, + pub average: f32, + pub bounding_box: Option<[f32; 4]>, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Statistics { + pub file_path: String, + pub file_size: f64, + pub schema: String, + pub page_size: Option, + pub level_details: Vec, +} + +impl Display for Statistics { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + writeln!(f, "File: {}", self.file_path).unwrap(); + writeln!(f, "FileSize: {:.2}MB", self.file_size).unwrap(); + writeln!(f, "Schema: {}", self.schema).unwrap(); + writeln!(f, "Page size: {} bytes", self.page_size.unwrap()).unwrap(); + writeln!( + f, + "|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|", + "zoom", "count", "smallest", "largest", "average", "bbox" + ) + .unwrap(); + + for t in &self.level_details { + let bbox_str = if let Some(b) = t.bounding_box { + format!("{:.2}, {:.2}, {:.2}, {:.2}", b[0], b[1], b[2], b[3]) + } else { + "".to_string() + }; + writeln!( + f, + "|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|{:^9}|", + t.zoom, + t.count, + format!("{:.2}KB", t.smallest), + format!("{:.2}KB", t.largest), + format!("{:.2}KB", t.average), + bbox_str + ) + .unwrap(); + } + Ok(()) + } +} + #[allow(clippy::trivially_copy_pass_by_ref)] fn serialize_ti(ti: &TileInfo, serializer: S) -> Result where @@ -107,7 +161,7 @@ pub struct Mbtiles { } impl Display for Mbtiles { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.filepath) } } @@ -223,7 +277,82 @@ impl Mbtiles { self.check_agg_tiles_hashes(&mut conn).await } } - + pub async fn statistics(&self, conn: &mut T) -> MbtResult + where + for<'e> &'e mut T: SqliteExecutor<'e>, + { + let file_size = + PathBuf::from(&self.filepath).metadata().unwrap().len() as f64 / 1024.0 / 1024.0; + let page_size_query = query!("PRAGMA page_size;"); + let tile_infos_query = query!( + r#"SELECT + zoom_level AS zoom, + count( ) AS count, + min( length( tile_data ) ) / 1024.0 AS smallest, + max( length( tile_data ) ) / 1024.0 AS largest, + avg( length( tile_data ) ) / 1024.0 AS average, + min(tile_column) as min_tile_x, + min(tile_row) as min_tile_y, + max(tile_column) as max_tile_x, + max(tile_row) as max_tile_y + FROM tiles + GROUP BY zoom_level"# + ); + let page_size = page_size_query.fetch_one(&mut *conn).await?.page_size; + let mb_type_string = match self.detect_type(&mut *conn).await? { + MbtType::Flat => "flat", + MbtType::FlatWithHash => "flat with hash", + MbtType::Normalized { .. } => "normalized", + }; + let tile_infos_rows = tile_infos_query.fetch_all(&mut *conn).await?; + let tile_infos: Vec = tile_infos_rows + .into_iter() + .map(|r| { + let zoom = r.zoom.unwrap() as u8; + let count = r.count as u32; + if count == 0 { + LevelDetail { + zoom, + count, + smallest: 0.0, + largest: 0.0, + average: 0.0, + bounding_box: None, + } + } else { + let min_tile_x = r.min_tile_x.expect(""); + let min_tile_y = r.min_tile_y.expect(""); + let max_tile_x = r.max_tile_x.expect(""); + let max_tile_y = r.max_tile_y.expect(""); + let tile_length: f32 = 40075016.7 / (2_u32.pow(zoom as u32)) as f32; + + let smallest = r.smallest.unwrap_or(0.0) as f32; + let largest = r.largest.unwrap_or(0.0) as f32; + let average = r.average.unwrap_or(0.0) as f32; + let minx = -20037508.34 + min_tile_x as f32 * tile_length; + let miny = -20037508.34 + min_tile_y as f32 * tile_length; + let maxx = -20037508.34 + (max_tile_x as f32 + 1.0) * tile_length; + let maxy = -20037508.34 + (max_tile_y as f32 + 1.0) * tile_length; + let bbox: [f32; 4] = [minx, miny, maxx, maxy]; + LevelDetail { + zoom, + count, + smallest, + largest, + average, + bounding_box: Some(bbox), + } + } + }) + .collect(); + Ok(Statistics { + file_path: self.filepath.clone(), + file_size, + schema: mb_type_string.to_owned(), + page_size, + level_details: tile_infos, + }) + } /// Get the aggregate tiles hash value from the metadata table pub async fn get_agg_tiles_hash(&self, conn: &mut T) -> MbtResult> where @@ -849,4 +978,32 @@ mod tests { assert!(matches!(result, Err(MbtError::AggHashMismatch(..)))); Ok(()) } + + #[actix_rt::test] + async fn stat() -> MbtResult<()> { + let (mut conn, mbt) = open("../tests/fixtures/mbtiles/world_cities.mbtiles").await?; + let res = mbt.statistics(&mut conn).await?; + + assert_eq!(res.file_path, "../tests/fixtures/mbtiles/world_cities.mbtiles"); + assert_eq!(res.file_size, 0.046875); + assert_eq!(res.schema, "flat"); + assert_eq!(res.page_size, Some(4096)); + + assert_eq!(res.level_details.len(), 7); + + assert_eq!(res.level_details[0].zoom, 0); + assert_eq!(res.level_details[0].count, 1); + assert_eq!(res.level_details[0].smallest, 1.0810547); + assert_eq!(res.level_details[0].largest, 1.0810547); + assert_eq!(res.level_details[0].average, 1.0810547); + assert_eq!(res.level_details[0].bounding_box, Some([-20037508.0, -20037508.0, 20037508.0, 20037508.0])); + + assert_eq!(res.level_details[6].zoom, 6); + assert_eq!(res.level_details[6].count, 72); + assert_eq!(res.level_details[6].smallest, 0.0625); + assert_eq!(res.level_details[6].largest, 0.09472656); + assert_eq!(res.level_details[6].average, 0.06669108); + assert_eq!(res.level_details[6].bounding_box, Some([-13775787.0, -5009377.0, 20037508.0, 8766410.0])); + Ok(()) + } } diff --git a/tests/expected/mbtiles/help.txt b/tests/expected/mbtiles/help.txt index 55eb51149..a9a82cc90 100644 --- a/tests/expected/mbtiles/help.txt +++ b/tests/expected/mbtiles/help.txt @@ -4,6 +4,7 @@ Usage: mbtiles Commands: meta-all Prints all values in the metadata table in a free-style, unstable YAML format + stats Gets tile statistics from MBTiels file meta-get Gets a single value from the MBTiles metadata table meta-set Sets a single value in the MBTiles' file metadata table or deletes it if no value copy Copy tiles from one mbtiles file to another diff --git a/tests/expected/mbtiles/stats.txt b/tests/expected/mbtiles/stats.txt new file mode 100644 index 000000000..2f1d999bb --- /dev/null +++ b/tests/expected/mbtiles/stats.txt @@ -0,0 +1,13 @@ +File: ./tests/fixtures/mbtiles/world_cities.mbtiles +FileSize: 0.05MB +Schema: flat +Page size: 4096 bytes +| zoom | count |smallest | largest | average | bbox | +| 0 | 1 | 1.08KB | 1.08KB | 1.08KB |-20037508.00, -20037508.00, 20037508.00, 20037508.00| +| 1 | 4 | 0.16KB | 0.63KB | 0.36KB |-20037508.00, -20037508.00, 20037508.00, 20037508.00| +| 2 | 7 | 0.13KB | 0.48KB | 0.23KB |-20037508.00, -10018754.00, 20037508.00, 10018754.00| +| 3 | 17 | 0.07KB | 0.24KB | 0.13KB |-15028131.00, -5009377.00, 20037508.00, 10018754.00| +| 4 | 38 | 0.06KB | 0.17KB | 0.08KB |-15028131.00, -5009377.00, 20037508.00, 10018754.00| +| 5 | 57 | 0.06KB | 0.10KB | 0.07KB |-13775787.00, -5009377.00, 20037508.00, 8766410.00| +| 6 | 72 | 0.06KB | 0.09KB | 0.07KB |-13775787.00, -5009377.00, 20037508.00, 8766410.00| + diff --git a/tests/test.sh b/tests/test.sh index 83a399716..8f5a9ffc7 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -314,6 +314,7 @@ if [[ "$MBTILES_BIN" != "-" ]]; then $MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt" $MBTILES_BIN meta-all --help 2>&1 | tee "$TEST_OUT_DIR/meta-all_help.txt" $MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/meta-all.txt" + $MBTILES_BIN stats ./tests/fixtures/mbtiles/world_cities.mbtiles 2>&1 | tee "$TEST_OUT_DIR/stats.txt" $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt" $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt" $MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"