feat: rotate streams

.gitignore

@@ -20,3 +20,5 @@ www/assets/
 mediamtx/
 onnxruntime/
 capture/
+
+python/

Cargo.toml

@@ -32,29 +32,30 @@ default = [
 ]
 
 person_matting = ["bevy_ort", "ort", "ndarray"]
-pipeline = ["image", "rayon"]
+pipeline = ["image", "imageproc", "rayon"]
 yolo = ["bevy_ort", "ort", "ndarray"]
 
 
 [dependencies]
 anyhow = "1.0"
 async-compat = "0.2"
 bevy_args = "1.3"
-bevy_ort = { version = "0.7.2", optional = true, features = ["yolo_v8"] }
+bevy_ort = { version = "0.8", optional = true, features = ["yolo_v8"] }
 bytes = "1.5"
 clap = { version = "4.4", features = ["derive"] }
 futures = "0.3"
-image = { version = "0.24", optional = true }
+image = { version = "0.24", optional = true }         # update /w `bevy` crate
+imageproc = { version = "0.23.0", optional = true }   # update /w `image` crate
 ndarray = { version = "0.15", optional = true }
 openh264 = "0.5"
+png = "0.17.13"
 rayon = { version = "1.8", optional = true }
 serde = "1.0"
 serde_json = "1.0"
 serde_qs = "0.12"
 retina = "0.4"
 tokio = { version = "1.36", features = ["full"] }
 url = "2.5"
-png = "0.17.13"
 
 
 [dependencies.bevy]

README.md

@@ -98,17 +98,6 @@ view the [onshape model](https://cad.onshape.com/documents/20d4b522e97cda88fb785
 ![Alt text](docs/light_field_camera_onshape_transparent.webp)
 
 
-## setup rtsp streaming server
-
-it is useful to test the light field viewer with emulated camera streams
-
-### obs studio
-
-- install https://obsproject.com/
-- install rtsp plugin https://github.com/iamscottxu/obs-rtspserver/releases
-- tools > rtsp server > start server
-
-
 ## compatible bevy versions
 
 | `bevy_light_field`    | `bevy` |
@@ -122,6 +111,7 @@ it is useful to test the light field viewer with emulated camera streams
 - [modnet](https://github.com/ZHKKKe/MODNet)
 - [nersemble](https://github.com/tobias-kirschstein/nersemble)
 - [paddle_seg_matting](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.9/Matting/docs/quick_start_en.md)
+- [pose diffusion](https://github.com/facebookresearch/PoseDiffusion)
 - [ray diffusion](https://github.com/jasonyzhang/RayDiffusion)
 
 

assets/streams.json

@@ -2,19 +2,19 @@
     { "uri": "rtsp://192.168.1.21/stream/main", "transport": "Udp", "visible": true },
     { "uri": "rtsp://192.168.1.22/stream/main", "transport": "Udp", "visible": true, "person_detection": true },
 
-    { "uri": "rtsp://192.168.1.23/user=admin&password=admin123&channel=1&stream=0.sdp?", "visible": true },
-    { "uri": "rtsp://192.168.1.24/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.25/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.26/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.27/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.28/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.29/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.30/user=admin&password=admin123&channel=1&stream=0.sdp?" },
+    { "uri": "rtsp://192.168.1.23/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 45.0, "visible": true },
+    { "uri": "rtsp://192.168.1.24/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 55.0 },
+    { "uri": "rtsp://192.168.1.25/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 90.0 },
+    { "uri": "rtsp://192.168.1.26/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -130.0 },
+    { "uri": "rtsp://192.168.1.27/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -90.0 },
+    { "uri": "rtsp://192.168.1.28/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -135.0 },
+    { "uri": "rtsp://192.168.1.29/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 125.0 },
+    { "uri": "rtsp://192.168.1.30/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -50.0 },
 
-    { "uri": "rtsp://192.168.1.31/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.32/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.33/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.34/user=admin&password=admin123&channel=1&stream=0.sdp?" },
-    { "uri": "rtsp://192.168.1.35/user=admin&password=admin123&channel=1&stream=0.sdp?" },
+    { "uri": "rtsp://192.168.1.31/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 },
+    { "uri": "rtsp://192.168.1.32/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 },
+    { "uri": "rtsp://192.168.1.33/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 },
+    { "uri": "rtsp://192.168.1.34/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 135.0 },
+    { "uri": "rtsp://192.168.1.35/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -45.0 },
     { "uri": "rtsp://192.168.1.36/user=admin&password=admin123&channel=1&stream=0.sdp?" }
 ]

src/lib.rs

@@ -1,10 +1,9 @@
 use bevy::prelude::*;
-
-#[cfg(feature = "person_matting")]
-pub mod matting;
+use bevy_ort::BevyOrtPlugin;
 
 pub mod ffmpeg;
 pub mod materials;
+pub mod matting;
 pub mod mp4;
 pub mod person_detect;
 pub mod pipeline;
@@ -18,6 +17,8 @@ pub struct LightFieldPlugin {
 
 impl Plugin for LightFieldPlugin {
     fn build(&self, app: &mut App) {
+        app.add_plugins(BevyOrtPlugin);
+
         app.add_plugins(materials::StreamMaterialsPlugin);
         app.add_plugins(person_detect::PersonDetectPlugin);
         app.add_plugins(pipeline::PipelinePlugin);

src/matting.rs

@@ -4,11 +4,9 @@ use bevy::{
     tasks::{block_on, futures_lite::future, AsyncComputeTaskPool, Task},
 };
 use bevy_ort::{
-    BevyOrtPlugin,
-    inputs,
     models::modnet::{
-        images_to_modnet_input,
-        modnet_output_to_luma_images,
+        Modnet,
+        modnet_inference,
     },
     Onnx,
 };
@@ -45,27 +43,20 @@ impl MattingPlugin {
 
 impl Plugin for MattingPlugin {
     fn build(&self, app: &mut App) {
-        app.add_plugins(BevyOrtPlugin);
         app.register_type::<MattedStream>();
-        app.init_resource::<Modnet>();
         app.insert_resource(self.max_inference_size.clone());
         app.add_systems(Startup, load_modnet);
         app.add_systems(Update, matting_inference);
     }
 }
 
 
-#[derive(Resource, Default)]
-pub struct Modnet {
-    pub onnx: Handle<Onnx>,
-}
-
 
 fn load_modnet(
     asset_server: Res<AssetServer>,
     mut modnet: ResMut<Modnet>,
 ) {
-    let modnet_handle: Handle<Onnx> = asset_server.load("modnet_photographic_portrait_matting.onnx");
+    let modnet_handle: Handle<Onnx> = asset_server.load("models/modnet_photographic_portrait_matting.onnx");
     modnet.onnx = modnet_handle;
 }
 
@@ -107,7 +98,8 @@ fn matting_inference(
         .map(|(_, matted_stream)| {
             let input = images.get(matted_stream.input.clone()).unwrap();
             let output = (matted_stream.output.clone(), matted_stream.material.clone());
-            (input, output)
+
+            (input.clone(), output)
         })
         .unzip();
 
@@ -116,30 +108,27 @@ fn matting_inference(
         return;
     }
 
-    let input = images_to_modnet_input(
-        inputs.as_slice(),
-        inference_size.0.into(),
-    );
-
     if onnx_assets.get(&modnet.onnx).is_none() {
         return;
     }
 
     let onnx = onnx_assets.get(&modnet.onnx).unwrap();
     let session_arc = onnx.session.clone();
 
+    let inference_size = inference_size.0.into();
+
     let task = thread_pool.spawn(async move {
+        let inputs = inputs.iter().collect::<Vec<_>>();
+
         let mask_images: Result<Vec<Image>, String> = (|| {
             let session_lock = session_arc.lock().map_err(|e| e.to_string())?;
             let session = session_lock.as_ref().ok_or("failed to get session from ONNX asset")?;
 
-            let input_values = inputs!["input" => input.view()].map_err(|e| e.to_string())?;
-            let outputs = session.run(input_values).map_err(|e| e.to_string());
-
-            let binding = outputs.ok().unwrap();
-            let output_value: &ort::Value = binding.get("output").unwrap();
-
-            Ok(modnet_output_to_luma_images(output_value))
+            Ok(modnet_inference(
+                session,
+                inputs.as_slice(),
+                inference_size,
+            ))
         })();
 
         match mask_images {