diff --git a/Cargo.lock b/Cargo.lock index bf3b8c6..6004c23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,6 +98,7 @@ checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" name = "app" version = "0.0.1" dependencies = [ + "bitcode", "cesiumtiles", "cesiumtiles-gltf", "cesiumtiles-gltf-json", @@ -118,12 +119,42 @@ dependencies = [ "thiserror", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "bitcode" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1bce7608560cd4bf0296a4262d0dbf13e6bcec5ff2105724c8ab88cc7fc784" +dependencies = [ + "arrayvec", + "bitcode_derive", + "bytemuck", + "glam", + "serde", +] + +[[package]] +name = "bitcode_derive" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a539389a13af092cd345a2b47ae7dec12deb306d660b2223d25cd3419b253ebe" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "bitflags" version = "2.6.0" @@ -136,6 +167,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytemuck" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" + [[package]] name = "byteorder" version = "1.5.0" @@ -378,6 +415,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glam" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc46dd3ec48fdd8e693a98d2b8bafae273a2d54c1de02a2a7e3d57d501f39677" + [[package]] name = "glob" version = "0.3.1" @@ -556,6 +599,7 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" name = "pcd-core" version = "0.0.1" dependencies = [ + "bitcode", "projection-transform", "serde", ] diff --git a/app/Cargo.toml b/app/Cargo.toml index ec2217c..9f11d19 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -23,3 +23,4 @@ serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.133" glob = "0.3.1" tempfile = "3.14.0" +bitcode = "0.6.3" diff --git a/app/src/main.rs b/app/src/main.rs index 4d4e173..89466d4 100644 --- a/app/src/main.rs +++ b/app/src/main.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::ffi::OsStr; -use std::fs::{File, OpenOptions}; -use std::io::{BufRead as _, BufReader, BufWriter, Write}; +use std::fs::File; +use std::io::{BufWriter, Write}; use std::{ fs, path::{Path, PathBuf}, @@ -103,32 +103,18 @@ fn write_points_to_tile( fs::create_dir_all(tile_path.parent().unwrap())?; - let file = OpenOptions::new() - .create(true) - .append(true) - .open(&tile_path)?; - + let file = File::create(tile_path)?; let mut writer = BufWriter::new(file); - for p in points { - let line = serde_json::to_string(p).unwrap(); - writeln!(writer, "{}", line)?; - } + let encoded = bitcode::encode(points); + writer.write_all(&encoded)?; Ok(()) } fn read_points_from_tile(file_path: &Path) -> std::io::Result> { - let file = File::open(file_path)?; - let reader = BufReader::new(file); - - let mut points = Vec::new(); - for line in reader.lines() { - let line = line?; - let p: Point = serde_json::from_str(&line).unwrap(); - points.push(p); - } - + let buf = std::fs::read(file_path)?; + let points = bitcode::decode(&buf).unwrap(); Ok(points) } @@ -176,28 +162,24 @@ fn aggregate_zoom_level(base_path: &Path, z: u8) -> std::io::Result<()> { let child_z = z + 1; let child_files = get_tile_list_for_zoom(base_path, child_z); + let mut parent_map: HashMap<(u8, u32, u32), Vec> = HashMap::new(); + for child_file in child_files { let (cz, cx, cy) = extract_tile_coords(&child_file); assert_eq!(cz, child_z); - let file = File::open(&child_file)?; - let reader = BufReader::new(file); - - let mut parent_map: HashMap<(u8, u32, u32), Vec> = HashMap::new(); - - for line in reader.lines() { - let line = line?; - let p: Point = serde_json::from_str(&line).unwrap(); + let points = read_points_from_tile(&child_file)?; + for p in points { let parent_x = cx / 2; let parent_y = cy / 2; let parent_coords = (z, parent_x, parent_y); parent_map.entry(parent_coords).or_default().push(p); } + } - for (parent_tile, pts) in parent_map { - write_points_to_tile(base_path, parent_tile, &pts)?; - } + for (parent_tile, pts) in parent_map { + write_points_to_tile(base_path, parent_tile, &pts)?; } Ok(()) @@ -353,39 +335,30 @@ fn main() { let min_zoom = args.min; let max_zoom = args.max; - let chunk_size = 1_000_000; - let mut start_idx = 0; - let total_points = transformed.points.len(); let tmp_dir_path = tempdir().unwrap(); log::info!("start tiling at max_zoom ({})...", max_zoom); let start_local = std::time::Instant::now(); // calc tile coords for max_zoom - while start_idx < total_points { - let end_idx = (start_idx + chunk_size).min(total_points); - let chunk = &transformed.points[start_idx..end_idx]; - start_idx = end_idx; - - // calc tile coords - let tile_pairs: Vec<((u8, u32, u32), Point)> = chunk - .par_iter() - .map(|p| { - let tile_coords = tiling::scheme::zxy_from_lng_lat(max_zoom, p.x, p.y); - (tile_coords, p.clone()) - }) - .collect(); - - // grouping by tile - let mut tile_map: HashMap<(u8, u32, u32), Vec> = HashMap::new(); - for (tile, pt) in tile_pairs { - tile_map.entry(tile).or_default().push(pt); - } + let tile_pairs: Vec<((u8, u32, u32), Point)> = transformed + .points + .par_iter() + .map(|p| { + let tile_coords = tiling::scheme::zxy_from_lng_lat(max_zoom, p.x, p.y); + (tile_coords, p.clone()) + }) + .collect(); - // export to tile files - for (tile, pts) in tile_map { - write_points_to_tile(tmp_dir_path.path(), tile, &pts).unwrap(); - } + // grouping by tile + let mut tile_map: HashMap<(u8, u32, u32), Vec> = HashMap::new(); + for (tile, pt) in tile_pairs { + tile_map.entry(tile).or_default().push(pt); + } + + // export to tile files + for (tile, pts) in tile_map { + write_points_to_tile(tmp_dir_path.path(), tile, &pts).unwrap(); } log::info!("Finish tiling at max_zoom in {:?}", start_local.elapsed()); diff --git a/pcd-core/Cargo.toml b/pcd-core/Cargo.toml index 6b263e8..8733f2a 100644 --- a/pcd-core/Cargo.toml +++ b/pcd-core/Cargo.toml @@ -4,5 +4,6 @@ version.workspace = true edition = "2021" [dependencies] +bitcode = "0.6.3" projection-transform = { path = "../projection-transform" } serde = "1.0.215" diff --git a/pcd-core/src/pointcloud/point.rs b/pcd-core/src/pointcloud/point.rs index bd42948..6fb357b 100644 --- a/pcd-core/src/pointcloud/point.rs +++ b/pcd-core/src/pointcloud/point.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; +use bitcode::{Decode, Encode}; use projection_transform::crs::EpsgCode; -use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Decode, Encode)] pub struct PointAttributes { pub intensity: Option, pub return_number: Option, @@ -15,7 +15,7 @@ pub struct PointAttributes { pub gps_time: Option, } -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Decode, Encode)] pub struct Color { pub r: u16, pub g: u16, @@ -25,7 +25,7 @@ pub struct Color { // LAS data coordinates are expressed in u32 format // The actual coordinates are calculated based on a combination of scale and offset, as follows // x = (x * scale[0]) + offset[0] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Decode, Encode)] pub struct Point { pub x: f64, pub y: f64, diff --git a/pcd-exporter/src/cesiumtiles.rs b/pcd-exporter/src/cesiumtiles.rs index 9f17ef2..f7c6445 100644 --- a/pcd-exporter/src/cesiumtiles.rs +++ b/pcd-exporter/src/cesiumtiles.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use pcd_core::pointcloud::point::{Point, PointCloud}; -use rayon::iter::{IntoParallelRefIterator as _, ParallelIterator as _}; use tinymvt::TileZXY; use crate::tiling::{self, TileContent};