diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..19ab3412 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Generated by Cargo +# will have compiled files and executables +**/target + +# Ignore all Cargo.lock but one at top-level, since it's committed in git. +*/**/Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# CLion project directory +.idea + +# Emacs temps +*~ + +# MacOS Related +.DS_Store + +# Output files +outfile.png +output.dot diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..199f3307 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,32 @@ +# +# Copyright (c) 2017, 2021 ADLINK Technology Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +# +# Contributors: +# ADLINK zenoh team, +# +[workspace] + +members = [ + "zenoh-flow", + "zenoh-flow-derive", + "zenoh-flow-examples", +] + +[profile.dev] +debug=true +opt-level = 0 + + +[profile.release] +debug=true +lto="fat" +codegen-units=1 +opt-level=3 +panic="abort" diff --git a/README.md b/README.md index 0d155ddf..1b6a8b87 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,184 @@ -# Eclipse zenoh flow +# Eclipse Zenoh Flow -zenoh-flow aims at providing a zenoh-based data-flow programming framework for computations that span from the cloud to the device. \ No newline at end of file +Zenoh Flow aims at providing a Zenoh-based dataflow programming framework for computations that span from the cloud to the device. + +:warning: **This software is still in alpha status and should _not_ be used in production. Breaking changes are likely to happen and the API is not stable.** + +----------- +## Description + +Users can describe a dataflow "pipeline" that should run on one or multiple Zenoh Flow instances via a `yaml` file. This file constitutes the entry point of Zenoh Flow. + +A pipeline is composed of set of _sources_ — producing data, _operators_ — computing over the data, and _sinks_ — consuming the resulting data. These components are _dynamically_ loaded at runtime. + +The different instances leverage Zenoh’s publish-subscribe model to communicate in a transparent manner for the user, i.e. without any configuration or intervention as Zenoh is built-in. + +We provide several working examples that illustrate how to author components and the yaml file describing a dataflow pipeline. + +----------- +## How to build it + +Install [Cargo and Rust](https://doc.rust-lang.org/cargo/getting-started/installation.html). Zenoh Flow can be successfully compiled with Rust stable (>= 1.5.1), so no special configuration is required — except for certain examples. + +To build Zenoh Flow, just type the following command after having followed the previous instructions: + +```bash +$ cargo build --release +``` + +----------- +## How to run + +If you launched the previous command, the Zenoh Flow runtime is located in `target/release/runtime`. This executable expects the following arguments: + +- the path of the dataflow graph to execute: `--graph-file zenoh-flow-examples/graphs/fizz_buzz_pipeline.yaml`, +- a name for the runtime: `--runtime foo`. + +The graph describes the different components composing the dataflow. Although mandatory, the name of the runtime is used to "deploy" the graph on different "runtime instances" (see the related examples). + +----------- +## Examples + +### FizzBuzz + +First, compile the relevant examples: + +```bash +cargo build --example manual-source --example example-fizz --example example-buzz --example generic-sink +``` + +This will create, depending on your OS, the libraries that the pipeline will fetch. + +#### Single runtime + +To run all components on the same Zenoh Flow runtime: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/fizz_buzz_pipeline.yaml --runtime foo +``` + +_Note: in that particular case the `--runtime foo` is discarded._ + +#### Multiple runtimes + +In a first machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/fizz-buzz-multiple-runtimes.yaml --runtime foo +``` + +In a second machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/fizz-buzz-multiple-runtimes.yaml --runtime bar +``` + +:warning: If you change the name of the runtime in the yaml file, the name(s) passed as argument of the previous commands must be changed accordingly. + +:warning: Without configuration, the different machines need to be on the _same local network_ for this example to work. See how to add a [Zenoh router](https://zenoh.io/docs/getting-started/key-concepts/#zenoh-router) if you want to connect them through the internet. + +--- + +### OpenCV FaceDetection - Haarcascades + +:warning: This example works only on Linux and it require OpenCV to be installed, please follow the instruction on the [OpenCV documentation](https://docs.opencv.org/4.5.2/d7/d9f/tutorial_linux_install.html) to install it. + +:warning: You need a machine equipped of a webcam in order to run this example. + +First, compile the relevant examples: + +```bash +cargo build --example camera-source --example face-detection --example video-sink +``` + +This will create, depending on your OS, the libraries that the pipeline will fetch. + +#### Single runtime + +To run all components on the same Zenoh Flow runtime: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/face_detection.yaml --runtime foo +``` + +_Note: in that particular case the `--runtime foo` is discarded._ + +#### Multiple runtimes + +In a first machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml --runtime gigot +``` + +In a second machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml --runtime nuc +``` + +In a third machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml --runtime leia +``` + +:warning: If you change the name of the runtime in the yaml file, the name(s) passed as argument of the previous commands must be changed accordingly. + +:warning: Without configuration, the different machines need to be on the _same local network_ for this example to work. See how to add a [Zenoh router](https://zenoh.io/docs/getting-started/key-concepts/#zenoh-router) if you want to connect them through the internet. + +--- + +### OpenCV Object Detection - Deep Neural Network - CUDA powered + +:warning: This example works only on Linux and it require OpenCV with CUDA enabled to be installed, please follow the instruction on [this gits](https://gist.github.com/raulqf/f42c718a658cddc16f9df07ecc627be7) to install it. + +:warning: This example works only on Linux and it require a **CUDA** capable **NVIDIA GPU**, as well as NVIDIA CUDA and CuDNN to be installed, please follow [CUDA instructions](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) and [CuDNN instructions](https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html). + +:warning: You need a machine equipped of a webcam in order to run this example. + +:warning: You need to download a YOLOv3 configuration, weights and classes, you can use the ones from [this GitHub repository](https://github.com/sthanhng/yoloface). + +First, compile the relevant examples: + +```bash +cargo build --example camera-source --example object-detection-dnn --example video-sink +``` + +This will create, depending on your OS, the libraries that the pipeline will fetch. + +Then please update the files `zenoh-flow-examples/graphs/dnn-object-detection.yaml` and `zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml` by changing the `neural-network`, `network-weights`, and `network-classes` to match the absolute path of your *Neural Network* configuration + +#### Single runtime + +To run all components on the same Zenoh Flow runtime: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/dnn-object-detection.yaml --runtime foo +``` + +_Note: in that particular case the `--runtime foo` is discarded._ + +#### Multiple runtimes + +In a first machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml --runtime foo +``` + +In a second machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml --runtime cuda +``` + +In a third machine, run: + +```bash +./target/release/runtime --graph-file zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml --runtime bar +``` + +:warning: If you change the name of the runtime in the yaml file, the name(s) passed as argument of the previous commands must be changed accordingly. + +:warning: Without configuration, the different machines need to be on the _same local network_ for this example to work. See how to add a [Zenoh router](https://zenoh.io/docs/getting-started/key-concepts/#zenoh-router) if you want to connect them through the internet. diff --git a/zenoh-flow-derive/Cargo.toml b/zenoh-flow-derive/Cargo.toml new file mode 100644 index 00000000..bb746f9b --- /dev/null +++ b/zenoh-flow-derive/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "zenoh-flow-derive" +version = "0.0.1" +repository = "https://github.com/eclipse-zenoh/zenoh-flow" +homepage = "http://zenoh.io" +authors = ["kydos ", + "gabrik ", + "Julien Loudet ",] +edition = "2018" +license = " EPL-2.0 OR Apache-2.0" +categories = ["network-programming"] +description = "Zenoh-Flow: zenoh-based data-flow programming framework for computations that span from the cloud to the device." +readme = "README.md" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +# To build with debug on macros: RUSTFLAGS="-Z macro-backtrace" + +[dependencies] +async-std = { version = "=1.9.0", features = ["attributes"] } +futures = "0.3.5" +syn-serde = { version = "0.2", features = ["json"] } +syn = { version = "1.0.11", features = ["full"] } +quote = "1.0.2" +proc-macro2 = "1.0.6" +serde_derive = "1.0.55" +serde = { version = "1.0.55", features = ["derive"] } +darling = "0.13.0" +Inflector = "0.11.4" +proc-macro-error = "1.0.4" + + +[dev-dependencies] +env_logger = "0.9" + + +[lib] +proc-macro = true + + diff --git a/zenoh-flow-derive/src/lib.rs b/zenoh-flow-derive/src/lib.rs new file mode 100644 index 00000000..778d2feb --- /dev/null +++ b/zenoh-flow-derive/src/lib.rs @@ -0,0 +1,57 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +#[proc_macro_derive(ZFData)] +pub fn zf_data_derive(input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + let ident = &ast.ident; + let gen = quote! { + + #[typetag::serde] + impl zenoh_flow::DataTrait for #ident { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_mut_any(&mut self) -> &mut dyn std::any::Any { + self + } + } + }; + gen.into() +} + +#[proc_macro_derive(ZFState)] +pub fn zf_state_derive(input: TokenStream) -> TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + let ident = &ast.ident; + let gen = quote! { + + #[typetag::serde] + impl zenoh_flow::StateTrait for #ident { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_mut_any(&mut self) -> &mut dyn std::any::Any { + self + } + } + }; + gen.into() +} diff --git a/zenoh-flow-examples/Cargo.toml b/zenoh-flow-examples/Cargo.toml new file mode 100644 index 00000000..994d4955 --- /dev/null +++ b/zenoh-flow-examples/Cargo.toml @@ -0,0 +1,91 @@ +[package] +name = "zenoh-flow-examples" +version = "0.1.0" +repository = "https://github.com/eclipse-zenoh/zenoh-flow" +homepage = "http://zenoh.io" +authors = ["kydos ", + "gabrik ", + "Julien Loudet ",] +edition = "2018" +license = " EPL-2.0 OR Apache-2.0" +categories = ["network-programming"] +description = "Zenoh-Flow: zenoh-based data-flow programming framework for computations that span from the cloud to the device." +readme = "README.md" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +zenoh-flow = { path = "../zenoh-flow" } +env_logger = "0.9" +serde = { version = "1.0.55", features = ["derive"] } +opencv = "0.53.0" +async-std = { version = "=1.9.0", features = ["attributes"] } +typetag = "0.1" +log = "0.4" +bincode = "1" +zenoh = { git = "https://github.com/eclipse-zenoh/zenoh.git", branch = "master", default-features = false, features = ["transport_tcp","transport_udp"] } + +[dev-dependencies] +rand = "0.8.0" +opencv = "0.53.0" +serde = { version = "1.0.55", features = ["derive"] } +async-std = { version = "=1.9.0", features = ["attributes","unstable"] } +futures = "0.3.5" +flume = "0.10" +serde_json = "1.0" + + +[[example]] +name = "example-fizz" +crate-type = ["cdylib"] + +[[example]] +name = "example-buzz" +crate-type = ["cdylib"] + +[[example]] +name = "random-source" +crate-type = ["cdylib"] + +[[example]] +name = "counter-source" +crate-type = ["cdylib"] + +[[example]] +name = "generic-sink" +crate-type = ["cdylib"] + +[[example]] +name = "zenoh-sink" +crate-type = ["cdylib"] + +[[example]] +name = "manual-source" +crate-type = ["cdylib"] + +[[example]] +name = "sum-and-send" +crate-type = ["cdylib"] + +[[example]] +name = "camera-source" +crate-type = ["cdylib"] + +[[example]] +name = "face-detection" +crate-type = ["cdylib"] + +[[example]] +name = "object-detection-dnn" +crate-type = ["cdylib"] + +[[example]] +name = "video-sink" +crate-type = ["cdylib"] + +[[example]] +name = "example-select" + +[[example]] +name = "video-pipeline" + diff --git a/zenoh-flow-examples/examples/camera-source.rs b/zenoh-flow-examples/examples/camera-source.rs new file mode 100644 index 00000000..530d61cd --- /dev/null +++ b/zenoh-flow-examples/examples/camera-source.rs @@ -0,0 +1,183 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::{Arc, Mutex}; +use opencv::{core, prelude::*, videoio}; +use std::collections::HashMap; +use zenoh_flow::{ + downcast_mut, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnOutputRule, FnSourceRun, FutRunResult, RunResult, SourceTrait, StateTrait, + ZFContext, ZFError, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_spin_lock, +}; +use zenoh_flow_examples::{ZFBytes, ZFOpenCVBytes}; + +#[derive(Debug)] +struct CameraSource { + pub state: CameraState, +} + +#[derive(Clone)] +struct InnerCameraAccess { + pub camera: Arc>, + pub encode_options: Arc>, +} + +#[derive(Serialize, Deserialize, ZFState, Clone)] +struct CameraState { + #[serde(skip_serializing, skip_deserializing)] + pub inner: Option, + + pub resolution: (i32, i32), + pub delay: u64, +} + +// because of opencv +impl std::fmt::Debug for CameraState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "CameraState: resolution:{:?} delay:{:?}", + self.resolution, self.delay + ) + } +} + +static SOURCE: &str = "Frame"; + +impl CameraSource { + fn new(configuration: HashMap) -> Self { + let configured_camera = configuration.get("camera").unwrap(); + let configured_resolution = match configuration.get("resolution") { + Some(res) => { + let v = res.split("x").collect::>(); + (v[0].parse::().unwrap(), v[1].parse::().unwrap()) + } + None => (800, 600), + }; + + let delay = match configuration.get("fps") { + Some(fps) => { + let fps = fps.parse::().unwrap(); + let delay: f64 = 1f64 / fps; + (delay * 1000f64) as u64 + } + None => 40, + }; + + let mut camera = + videoio::VideoCapture::from_file(configured_camera, videoio::CAP_ANY).unwrap(); // 0 is the default camera + let opened = videoio::VideoCapture::is_opened(&camera).unwrap(); + if !opened { + panic!("Unable to open default camera!"); + } + let encode_options = opencv::types::VectorOfi32::new(); + let inner = InnerCameraAccess { + camera: Arc::new(Mutex::new(camera)), + encode_options: Arc::new(Mutex::new(encode_options)), + }; + + let state = CameraState { + inner: Some(inner), + resolution: configured_resolution, + delay, + }; + + Self { state } + } + + async fn run_1(ctx: ZFContext) -> RunResult { + let mut results: HashMap> = HashMap::new(); + + let mut guard = ctx.async_lock().await; + let mut _state = downcast_mut!(CameraState, guard.state).unwrap(); //downcasting to right type + + let inner = _state.inner.as_ref().unwrap(); + { + // just to force rust understand that the not sync variables are dropped before the sleep + let mut cam = zf_spin_lock!(inner.camera); + let encode_options = zf_spin_lock!(inner.encode_options); + + let mut frame = core::Mat::default(); + cam.read(&mut frame).unwrap(); + + let mut reduced = Mat::default(); + opencv::imgproc::resize( + &frame, + &mut reduced, + opencv::core::Size::new(_state.resolution.0, _state.resolution.0), + 0.0, + 0.0, + opencv::imgproc::INTER_LINEAR, + ) + .unwrap(); + + let mut buf = opencv::types::VectorOfu8::new(); + opencv::imgcodecs::imencode(".jpg", &reduced, &mut buf, &encode_options).unwrap(); + + // let data = ZFOpenCVBytes {bytes: Mutex::new(RefCell::new(buf))}; + + let data = ZFBytes { + bytes: buf.to_vec(), + }; + + results.insert(String::from(SOURCE), zf_data!(data)); + + drop(cam); + drop(encode_options); + drop(frame); + drop(reduced); + } + + async_std::task::sleep(std::time::Duration::from_millis(_state.delay.clone())).await; + + Ok(results) + } +} + +impl SourceTrait for CameraSource { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(|ctx: ZFContext| -> FutRunResult { Box::pin(Self::run_1(ctx)) }), + _ => panic!("No way"), + } + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(zenoh_flow::default_output_rule) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +// //Also generated by macro +zenoh_flow::export_source!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + match configuration { + Some(config) => { + Ok(Box::new(CameraSource::new(config)) as Box) + } + None => Err(ZFError::MissingConfiguration), + } +} diff --git a/zenoh-flow-examples/examples/counter-source.rs b/zenoh-flow-examples/examples/counter-source.rs new file mode 100644 index 00000000..c82cfc29 --- /dev/null +++ b/zenoh-flow-examples/examples/counter-source.rs @@ -0,0 +1,84 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use zenoh_flow::{ + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnOutputRule, FnSourceRun, FutRunResult, RunResult, SourceTrait, StateTrait, + ZFContext, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_empty_state, +}; +use zenoh_flow_examples::RandomData; + +static SOURCE: &str = "Number"; + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Serialize, Deserialize, Debug, ZFState)] +struct CountSource {} + +impl CountSource { + fn new(configuration: Option>) -> Self { + match configuration { + Some(conf) => { + let initial = conf.get("initial").unwrap().parse::().unwrap(); + COUNTER.store(initial, Ordering::AcqRel); + CountSource {} + } + None => CountSource {}, + } + } + + async fn run_1(_ctx: ZFContext) -> RunResult { + let mut results: HashMap> = HashMap::new(); + let d = RandomData { + d: COUNTER.fetch_add(1, Ordering::AcqRel), + }; + results.insert(String::from(SOURCE), zf_data!(d)); + async_std::task::sleep(std::time::Duration::from_secs(1)).await; + Ok(results) + } +} + +impl SourceTrait for CountSource { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(|ctx: ZFContext| -> FutRunResult { Box::pin(Self::run_1(ctx)) }), + _ => panic!("No way"), + } + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(zenoh_flow::default_output_rule) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +// //Also generated by macro +zenoh_flow::export_source!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + Ok(Box::new(CountSource::new(configuration)) as Box) +} diff --git a/zenoh-flow-examples/examples/example-buzz.rs b/zenoh-flow-examples/examples/example-buzz.rs new file mode 100644 index 00000000..1a2f59ae --- /dev/null +++ b/zenoh-flow-examples/examples/example-buzz.rs @@ -0,0 +1,103 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use zenoh_flow::{ + export_operator, get_input, + runtime::message::ZFMessage, + types::{ + DataTrait, FnInputRule, FnOutputRule, FnRun, InputRuleResult, OperatorTrait, + OutputRuleResult, RunResult, StateTrait, ZFInput, ZFResult, + }, + zf_data, zf_empty_state, Token, ZFContext, ZFLinkId, +}; +use zenoh_flow_examples::{ZFString, ZFUsize}; + +struct BuzzOperator; + +static LINK_ID_INPUT_INT: &str = "Int"; +static LINK_ID_INPUT_STR: &str = "Str"; +static LINK_ID_OUTPUT_STR: &str = "Str"; + +impl BuzzOperator { + fn input_rule(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + for token in inputs.values() { + match token { + Token::Ready(_) => continue, + Token::NotReady(_) => return Ok(false), + } + } + + Ok(true) + } + + fn run(_ctx: ZFContext, mut inputs: ZFInput) -> RunResult { + let mut results = HashMap::>::with_capacity(1); + + let mut buzz = get_input!(ZFString, String::from(LINK_ID_INPUT_STR), inputs)?.clone(); + + let value = get_input!(ZFUsize, String::from(LINK_ID_INPUT_INT), inputs)?; + + if value.0 % 3 == 0 { + buzz.0.push_str("Buzz"); + } + + results.insert(String::from(LINK_ID_OUTPUT_STR), zf_data!(buzz)); + + Ok(results) + } + + fn output_rule( + _ctx: ZFContext, + outputs: HashMap>, + ) -> OutputRuleResult { + let mut zf_outputs: HashMap> = HashMap::with_capacity(1); + + zf_outputs.insert( + String::from(LINK_ID_OUTPUT_STR), + Arc::new(ZFMessage::from_data( + outputs.get(LINK_ID_OUTPUT_STR).unwrap().clone(), + )), + ); + + Ok(zf_outputs) + } +} + +impl OperatorTrait for BuzzOperator { + fn get_input_rule(&self, _ctx: ZFContext) -> Box { + Box::new(BuzzOperator::input_rule) + } + + fn get_run(&self, _ctx: ZFContext) -> Box { + Box::new(BuzzOperator::run) + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(BuzzOperator::output_rule) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +export_operator!(register); + +extern "C" fn register( + _configuration: Option>, +) -> ZFResult> { + Ok(Box::new(BuzzOperator) as Box) +} diff --git a/zenoh-flow-examples/examples/example-fizz.rs b/zenoh-flow-examples/examples/example-fizz.rs new file mode 100644 index 00000000..33555e1f --- /dev/null +++ b/zenoh-flow-examples/examples/example-fizz.rs @@ -0,0 +1,111 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use zenoh_flow_examples::{ZFString, ZFUsize}; + +use zenoh_flow::{ + downcast, export_operator, get_input, + runtime::message::ZFMessage, + types::{ + DataTrait, FnInputRule, FnOutputRule, FnRun, InputRuleResult, OperatorTrait, + OutputRuleResult, RunResult, StateTrait, ZFInput, ZFResult, + }, + zf_data, zf_empty_state, Token, ZFContext, ZFError, ZFLinkId, +}; + +struct FizzOperator; + +static LINK_ID_INPUT_INT: &str = "Int"; +static LINK_ID_OUTPUT_INT: &str = "Int"; +static LINK_ID_OUTPUT_STR: &str = "Str"; + +impl FizzOperator { + fn input_rule(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + for token in inputs.values() { + match token { + Token::Ready(_) => continue, + Token::NotReady(_) => return Ok(false), + } + } + + Ok(true) + } + + fn run(_ctx: ZFContext, mut inputs: ZFInput) -> RunResult { + let mut results = HashMap::>::with_capacity(2); + + let mut fizz = ZFString::from(""); + + let zfusize = get_input!(ZFUsize, String::from(LINK_ID_INPUT_INT), inputs)?; + + if zfusize.0 % 2 == 0 { + fizz = ZFString::from("Fizz"); + } + + results.insert(String::from(LINK_ID_OUTPUT_INT), zf_data!(zfusize.clone())); + results.insert(String::from(LINK_ID_OUTPUT_STR), zf_data!(fizz)); + + Ok(results) + } + + fn output_rule( + _ctx: ZFContext, + outputs: HashMap>, + ) -> OutputRuleResult { + let mut zf_outputs: HashMap> = HashMap::with_capacity(2); + + zf_outputs.insert( + String::from(LINK_ID_OUTPUT_INT), + Arc::new(ZFMessage::from_data( + outputs.get(LINK_ID_OUTPUT_INT).unwrap().clone(), + )), + ); + zf_outputs.insert( + String::from(LINK_ID_OUTPUT_STR), + Arc::new(ZFMessage::from_data( + outputs.get(LINK_ID_OUTPUT_STR).unwrap().clone(), + )), + ); + + Ok(zf_outputs) + } +} + +impl OperatorTrait for FizzOperator { + fn get_input_rule(&self, _ctx: ZFContext) -> Box { + Box::new(FizzOperator::input_rule) + } + + fn get_run(&self, _ctx: ZFContext) -> Box { + Box::new(FizzOperator::run) + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(FizzOperator::output_rule) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +export_operator!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + Ok(Box::new(FizzOperator) as Box) +} diff --git a/zenoh-flow-examples/examples/example-select.rs b/zenoh-flow-examples/examples/example-select.rs new file mode 100644 index 00000000..fd081360 --- /dev/null +++ b/zenoh-flow-examples/examples/example-select.rs @@ -0,0 +1,61 @@ +use async_std::sync::Arc; +use async_std::task::sleep; +use futures::future; +use std::time::Duration; +use zenoh_flow::runtime::graph::link::{link, ZFLinkSender}; +//use async_std::channel::{unbounded, Sender, Receiver}; + +async fn send(sender: ZFLinkSender, interveal: Duration, data: T) { + loop { + sender.send(Arc::new(data.clone())).await; + sleep(interveal).await; + } +} + +#[async_std::main] +async fn main() { + let (s1, mut r1) = link::(Some(10), String::from("0")); + let (s2, mut r2) = link::(Some(10), String::from("1")); + let (s3, mut r3) = link::(Some(10), String::from("2")); + let (s4, mut r4) = link::(Some(10), String::from("3")); + + let _h1 = async_std::task::spawn(async move { + send(s1, Duration::from_secs(1), 0u8).await; + }); + + let _h2 = async_std::task::spawn(async move { + send(s2, Duration::from_millis(250), 1u8).await; + }); + + let _h3 = async_std::task::spawn(async move { + send(s3, Duration::from_millis(750), 2u8).await; + }); + + let _h4 = async_std::task::spawn(async move { + send(s4, Duration::from_millis(500), 3u8).await; + }); + + loop { + let mut futs = vec![]; + + futs.push(r1.recv()); + futs.push(r2.recv()); + futs.push(r3.recv()); + futs.push(r4.recv()); + + while !futs.is_empty() { + match future::select_all(futs).await { + //this could be "slow" as suggested by LC + (Ok(v), _i, remaining) => { + println!("Link n. {:?} has terminated with {:?}", v.0, v.1); + futs = remaining; + } + (Err(e), i, remaining) => { + println!("Link index {:?} has got error {:?}", i, e); + futs = remaining; + } + } + } + println!("All link recv terminated, restarting...") + } +} diff --git a/zenoh-flow-examples/examples/face-detection.rs b/zenoh-flow-examples/examples/face-detection.rs new file mode 100644 index 00000000..739fe499 --- /dev/null +++ b/zenoh-flow-examples/examples/face-detection.rs @@ -0,0 +1,223 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::{Arc, Mutex}; +use rand::Rng; +use std::any::Any; +use std::cell::RefCell; +use std::collections::HashMap; +use std::io; +use zenoh_flow::{ + downcast, downcast_mut, get_input, + runtime::message::ZFMessage, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnInputRule, FnOutputRule, FnRun, InputRuleResult, OperatorTrait, + OutputRuleResult, RunResult, StateTrait, Token, ZFContext, ZFError, ZFInput, ZFLinkId, + ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_spin_lock, +}; +use zenoh_flow_examples::{ZFBytes, ZFOpenCVBytes}; + +use opencv::{core, highgui, imgproc, objdetect, prelude::*, types, videoio, Result}; + +static INPUT: &str = "Frame"; +static OUTPUT: &str = "Frame"; + +#[derive(Debug)] +struct FaceDetection { + pub state: FDState, +} + +#[derive(Clone)] +struct FDInnerState { + pub face: Arc>, + pub encode_options: Arc>, +} + +#[derive(Serialize, Deserialize, ZFState, Clone)] +struct FDState { + #[serde(skip_serializing, skip_deserializing)] + pub inner: Option, +} + +// because of opencv +impl std::fmt::Debug for FDState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "FDState:...",) + } +} + +impl FaceDetection { + fn new(configuration: HashMap) -> Self { + let neural_network = match configuration.get("neural-network") { + Some(net) => net, + None => "haarcascades/haarcascade_frontalface_alt.xml", + }; + + let xml = core::find_file(neural_network, true, false).unwrap(); + let mut face = objdetect::CascadeClassifier::new(&xml).unwrap(); + let encode_options = opencv::types::VectorOfi32::new(); + + let inner = Some(FDInnerState { + face: Arc::new(Mutex::new(face)), + encode_options: Arc::new(Mutex::new(encode_options)), + }); + let state = FDState { inner }; + + Self { state } + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub fn run_1(ctx: ZFContext, mut inputs: ZFInput) -> RunResult { + let mut results: HashMap> = HashMap::new(); + + let mut guard = ctx.lock(); //getting state + let _state = downcast!(FDState, guard.state).unwrap(); //downcasting to right type + + let inner = _state.inner.as_ref().unwrap(); + + let mut face = zf_spin_lock!(inner.face); + let encode_options = zf_spin_lock!(inner.encode_options); + + let data = get_input!(ZFBytes, String::from(INPUT), inputs).unwrap(); + + // Decode Image + let mut frame = opencv::imgcodecs::imdecode( + &opencv::types::VectorOfu8::from_iter(data.bytes.clone()), + opencv::imgcodecs::IMREAD_COLOR, + ) + .unwrap(); + + let mut gray = Mat::default(); + imgproc::cvt_color(&frame, &mut gray, imgproc::COLOR_BGR2GRAY, 0).unwrap(); + let mut reduced = Mat::default(); + imgproc::resize( + &gray, + &mut reduced, + core::Size { + width: 0, + height: 0, + }, + 0.25f64, + 0.25f64, + imgproc::INTER_LINEAR, + ) + .unwrap(); + let mut faces = types::VectorOfRect::new(); + face.detect_multi_scale( + &reduced, + &mut faces, + 1.1, + 2, + objdetect::CASCADE_SCALE_IMAGE, + core::Size { + width: 30, + height: 30, + }, + core::Size { + width: 0, + height: 0, + }, + ) + .unwrap(); + for face in faces { + let scaled_face = core::Rect { + x: face.x * 4, + y: face.y * 4, + width: face.width * 4, + height: face.height * 4, + }; + imgproc::rectangle( + &mut frame, + scaled_face, + core::Scalar::new(0f64, 255f64, -1f64, -1f64), + 10, + 1, + 0, + ) + .unwrap(); + } + + let mut buf = opencv::types::VectorOfu8::new(); + opencv::imgcodecs::imencode(".jpg", &frame, &mut buf, &encode_options).unwrap(); + + let data = ZFBytes { + bytes: buf.to_vec(), + }; + + results.insert(String::from(OUTPUT), zf_data!(data)); + + drop(face); + + Ok(results) + } + + pub fn or_1( + _ctx: ZFContext, + outputs: HashMap>, + ) -> OutputRuleResult { + let mut results = HashMap::new(); + for (k, v) in outputs { + // should be ZFMessage::from_data + results.insert(k, Arc::new(ZFMessage::from_data(v))); + } + Ok(results) + } +} + +impl OperatorTrait for FaceDetection { + fn get_input_rule(&self, ctx: ZFContext) -> Box { + Box::new(Self::ir_1) + } + + fn get_output_rule(&self, ctx: ZFContext) -> Box { + Box::new(Self::or_1) + } + + fn get_run(&self, ctx: ZFContext) -> Box { + Box::new(Self::run_1) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +// //Also generated by macro +zenoh_flow::export_operator!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + match configuration { + Some(config) => { + Ok(Box::new(FaceDetection::new(config)) as Box) + } + None => Ok(Box::new(FaceDetection::new(HashMap::new())) + as Box), + } +} diff --git a/zenoh-flow-examples/examples/generic-sink.rs b/zenoh-flow-examples/examples/generic-sink.rs new file mode 100644 index 00000000..246bcbd5 --- /dev/null +++ b/zenoh-flow-examples/examples/generic-sink.rs @@ -0,0 +1,74 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use std::collections::HashMap; +use zenoh_flow::{ + serde::{Deserialize, Serialize}, + types::{ + FnInputRule, FnSinkRun, FutSinkResult, InputRuleResult, SinkTrait, StateTrait, Token, + ZFContext, ZFInput, ZFLinkId, + }, + zenoh_flow_derive::ZFState, + zf_empty_state, ZFResult, +}; + +#[derive(Serialize, Deserialize, Debug, ZFState)] +struct ExampleGenericSink {} + +impl ExampleGenericSink { + pub fn ir_1(_ctx: ZFContext, _inputs: &mut HashMap) -> InputRuleResult { + Ok(true) + } + + pub async fn run_1(_ctx: ZFContext, inputs: ZFInput) -> ZFResult<()> { + println!("#######"); + for (k, v) in inputs.into_iter() { + println!("Example Generic Sink Received on LinkId {:?} -> {:?}", k, v); + } + println!("#######"); + Ok(()) + } +} + +impl SinkTrait for ExampleGenericSink { + fn get_input_rule(&self, ctx: ZFContext) -> Box { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(Self::ir_1), + _ => panic!("No way"), + } + } + + fn get_run(&self, ctx: ZFContext) -> FnSinkRun { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(|ctx: ZFContext, inputs: ZFInput| -> FutSinkResult { + Box::pin(Self::run_1(ctx, inputs)) + }), + _ => panic!("No way"), + } + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +zenoh_flow::export_sink!(register); + +extern "C" fn register( + _configuration: Option>, +) -> ZFResult> { + Ok(Box::new(ExampleGenericSink {}) as Box) +} diff --git a/zenoh-flow-examples/examples/manual-source.rs b/zenoh-flow-examples/examples/manual-source.rs new file mode 100644 index 00000000..eed4bbe9 --- /dev/null +++ b/zenoh-flow-examples/examples/manual-source.rs @@ -0,0 +1,72 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use std::{collections::HashMap, sync::Arc, usize}; + +use zenoh_flow::runtime::runner::ZFSourceRegistrarTrait; +use zenoh_flow::{ + types::{ + DataTrait, FnOutputRule, FnSourceRun, FutRunResult, RunResult, SourceTrait, StateTrait, + }, + zf_data, zf_empty_state, ZFContext, ZFError, ZFLinkId, ZFResult, +}; +use zenoh_flow_examples::ZFUsize; + +struct ManualSource; + +static LINK_ID_INPUT_INT: &str = "Int"; + +impl ManualSource { + async fn run(_ctx: ZFContext) -> RunResult { + let mut results: HashMap> = HashMap::with_capacity(1); + + println!("> Please input a number: "); + let mut number = String::new(); + async_std::io::stdin() + .read_line(&mut number) + .await + .expect("Could not read number."); + + let value: usize = match number.trim().parse() { + Ok(value) => value, + Err(_) => return Err(ZFError::GenericError), + }; + + results.insert(String::from(LINK_ID_INPUT_INT), zf_data!(ZFUsize(value))); + + Ok(results) + } +} + +impl SourceTrait for ManualSource { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun { + Box::new(|ctx: ZFContext| -> FutRunResult { Box::pin(Self::run(ctx)) }) + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(zenoh_flow::default_output_rule) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +zenoh_flow::export_source!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + Ok(Box::new(ManualSource {}) as Box) +} diff --git a/zenoh-flow-examples/examples/object-detection-dnn.rs b/zenoh-flow-examples/examples/object-detection-dnn.rs new file mode 100644 index 00000000..a8228738 --- /dev/null +++ b/zenoh-flow-examples/examples/object-detection-dnn.rs @@ -0,0 +1,369 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::{Arc, Mutex}; +use rand::Rng; +use std::any::Any; +use std::cell::RefCell; +use std::collections::HashMap; +use std::io; +use std::{ + fs::File, + io::{prelude::*, BufReader}, + path::Path, +}; +use zenoh_flow::{ + downcast, downcast_mut, get_input, + runtime::message::ZFMessage, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnInputRule, FnOutputRule, FnRun, InputRuleResult, OperatorTrait, + OutputRuleResult, RunResult, StateTrait, Token, ZFContext, ZFError, ZFInput, ZFLinkId, + ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_spin_lock, +}; +use zenoh_flow_examples::{ZFBytes, ZFOpenCVBytes}; + +use opencv::core::prelude::MatTrait; +use opencv::dnn::NetTrait; +use opencv::{core, highgui, imgproc, objdetect, prelude::*, types, videoio, Result}; +use std::time::{Duration, Instant}; + +static INPUT: &str = "Frame"; +static OUTPUT: &str = "Frame"; + +#[derive(Debug)] +struct ObjDetection { + pub state: FDState, +} +#[derive(Clone)] +struct FDInnerState { + pub dnn: Arc>, + pub classes: Arc>>, + pub encode_options: Arc>, + pub outputs: Arc>>, +} + +#[derive(Serialize, Deserialize, ZFState, Clone)] +struct FDState { + #[serde(skip_serializing, skip_deserializing)] + pub inner: Option, +} + +fn lines_from_file(filename: impl AsRef) -> Vec { + let file = File::open(filename).expect("no such file"); + let buf = BufReader::new(file); + buf.lines() + .map(|l| l.expect("Could not parse line")) + .collect() +} + +// because of opencv +impl std::fmt::Debug for FDState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "FDState:...",) + } +} + +impl ObjDetection { + fn init(configuration: HashMap) -> ZFResult { + let net_cfg = match configuration.get("neural-network") { + Some(net) => net, + None => return Err(ZFError::MissingConfiguration), + }; + + let net_weights = match configuration.get("network-weights") { + Some(weights) => weights, + None => return Err(ZFError::MissingConfiguration), + }; + + let net_classes = match configuration.get("network-classes") { + Some(classes) => classes, + None => return Err(ZFError::MissingConfiguration), + }; + + let classes = lines_from_file(net_classes); + + let mut net = opencv::dnn::read_net_from_darknet(net_cfg, net_weights).unwrap(); + let encode_options = opencv::types::VectorOfi32::new(); + + net.set_preferable_backend(opencv::dnn::DNN_BACKEND_CUDA) + .unwrap(); + net.set_preferable_target(opencv::dnn::DNN_TARGET_CUDA) + .unwrap(); + + let output_names = net.get_unconnected_out_layers_names().unwrap(); + + let inner = Some(FDInnerState { + dnn: Arc::new(Mutex::new(net)), + classes: Arc::new(Mutex::new(classes)), + encode_options: Arc::new(Mutex::new(encode_options)), + outputs: Arc::new(Mutex::new(output_names)), + }); + let state = FDState { inner }; + + Ok(Self { state }) + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub fn run_1(ctx: ZFContext, mut inputs: ZFInput) -> RunResult { + let scale = 1.0 / 255.0; + let mean = core::Scalar::new(0f64, 0f64, 0f64, 0f64); + + let mut results: HashMap> = HashMap::new(); + + let mut detections: opencv::types::VectorOfMat = core::Vector::new(); + + let mut guard = ctx.lock(); //getting state + let _state = downcast!(FDState, guard.state).unwrap(); //downcasting to right type + + let inner = _state.inner.as_ref().unwrap(); + + let mut net = zf_spin_lock!(inner.dnn); + let mut encode_options = zf_spin_lock!(inner.encode_options); + let mut classes = zf_spin_lock!(inner.classes); + let outputs = zf_spin_lock!(inner.outputs); + + let mut boxes: Vec> = vec![core::Vector::new(); classes.len()]; + let mut scores: Vec> = vec![core::Vector::new(); classes.len()]; + let mut indices: Vec> = vec![core::Vector::new(); classes.len()]; + + let colors: Vec = vec![ + core::Scalar::new(0f64, 255f64, 0f64, -1f64), + core::Scalar::new(255f64, 255f64, 0f64, -1f64), + core::Scalar::new(0f64, 255f64, 255f64, -1f64), + core::Scalar::new(255f64, 0f64, 0f64, -1f64), + ]; + + let data = get_input!(ZFBytes, String::from(INPUT), inputs).unwrap(); + + // Decode Image + let mut frame = opencv::imgcodecs::imdecode( + &opencv::types::VectorOfu8::from_iter(data.bytes.clone()), + opencv::imgcodecs::IMREAD_COLOR, + ) + .unwrap(); + + // create blob + let blob = opencv::dnn::blob_from_image( + &frame, + scale, + core::Size { + width: 512, + height: 512, //416 //608 + }, + mean, + true, + false, + opencv::core::CV_32F, //CV_32F + ) + .unwrap(); + + //set the input + net.set_input(&blob, "", 1.0, core::Scalar::new(0f64, 0f64, 0f64, 0f64)) + .unwrap(); + + //run the DNN + let now = Instant::now(); + net.forward(&mut detections, &outputs).unwrap(); + let elapsed = now.elapsed().as_micros(); + + // loop on the detected objects + for obj in detections { + let num_boxes = obj.rows(); + + for i in 0..num_boxes { + let x = obj.at_2d::(i, 0).unwrap() * frame.cols() as f32; + let y = obj.at_2d::(i, 1).unwrap() * frame.rows() as f32; + let width = obj.at_2d::(i, 2).unwrap() * frame.cols() as f32; + let height = obj.at_2d::(i, 3).unwrap() * frame.rows() as f32; + + let scaled_obj = core::Rect { + x: (x - width / 2.0) as i32, + y: (y - height / 2.0) as i32, + width: width as i32, + height: height as i32, + }; + + for c in 0..classes.len() { + let conf = *obj.at_2d::(i, 5 + (c as i32)).unwrap(); + if conf >= 0.4 { + boxes[c].push(scaled_obj); + scores[c].push(conf); + } + } + } + } + + //remove duplicates + for c in 0..classes.len() { + opencv::dnn::nms_boxes(&boxes[c], &scores[c], 0.0, 0.4, &mut indices[c], 1.0, 0) + .unwrap(); + } + + let mut detected = 0; + + // add boxes with score + for c in 0..classes.len() { + for i in &indices[c] { + let rect = boxes[c].get(i as usize).unwrap(); + let score = scores[c].get(i as usize).unwrap(); + + let color = colors[c % 4]; + + imgproc::rectangle( + &mut frame, rect, color, //green + 2, 1, 0, + ) + .unwrap(); + + let label = format!("{}: {}", classes[c], score); + let mut baseline = 0; + imgproc::get_text_size( + &label, + opencv::imgproc::FONT_HERSHEY_COMPLEX_SMALL, + 1.0, + 1, + &mut baseline, + ) + .unwrap(); + + imgproc::put_text( + &mut frame, + &label, + core::Point_::new(rect.x, rect.y - baseline - 5), + opencv::imgproc::FONT_HERSHEY_COMPLEX_SMALL, + 1.0, + color, //black + 2, + 8, + false, + ) + .unwrap(); + + detected += 1; + } + } + + // add label to frame with info + let label = format!("DNN Time: {} us - Detected: {}", elapsed, detected); + let mut baseline = 0; + + let bg_size = imgproc::get_text_size( + &label, + opencv::imgproc::FONT_HERSHEY_COMPLEX_SMALL, + 1.0, + 1, + &mut baseline, + ) + .unwrap(); + let rect = core::Rect { + x: 0, + y: 0, + width: bg_size.width, + height: bg_size.height + 10, + }; + + imgproc::rectangle( + &mut frame, + rect, + core::Scalar::new(0f64, 0f64, 0f64, -1f64), //black + imgproc::FILLED, + 1, + 0, + ) + .unwrap(); + imgproc::put_text( + &mut frame, + &label, + core::Point_::new(0, bg_size.height + 5), + opencv::imgproc::FONT_HERSHEY_COMPLEX_SMALL, + 1.0, + core::Scalar::new(255f64, 255f64, 0f64, -1f64), //yellow + 2, + 8, + false, + ) + .unwrap(); + + // encode and send + let mut buf = opencv::types::VectorOfu8::new(); + opencv::imgcodecs::imencode(".jpg", &frame, &mut buf, &encode_options).unwrap(); + + let data = ZFBytes { + bytes: buf.to_vec(), + }; + + results.insert(String::from(OUTPUT), zf_data!(data)); + + Ok(results) + } + + pub fn or_1( + _ctx: ZFContext, + outputs: HashMap>, + ) -> OutputRuleResult { + let mut results = HashMap::new(); + for (k, v) in outputs { + // should be ZFMessage::from_data + results.insert(k, Arc::new(ZFMessage::from_data(v))); + } + Ok(results) + } +} + +impl OperatorTrait for ObjDetection { + fn get_input_rule(&self, ctx: ZFContext) -> Box { + Box::new(Self::ir_1) + } + + fn get_output_rule(&self, ctx: ZFContext) -> Box { + Box::new(Self::or_1) + } + + fn get_run(&self, ctx: ZFContext) -> Box { + Box::new(Self::run_1) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +// //Also generated by macro +zenoh_flow::export_operator!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + match configuration { + Some(config) => { + Ok(Box::new(ObjDetection::init(config)?) as Box) + } + None => Ok(Box::new(ObjDetection::init(HashMap::new())?) + as Box), + } +} diff --git a/zenoh-flow-examples/examples/random-source.rs b/zenoh-flow-examples/examples/random-source.rs new file mode 100644 index 00000000..bcaa70d5 --- /dev/null +++ b/zenoh-flow-examples/examples/random-source.rs @@ -0,0 +1,66 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use zenoh_flow::{ + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnOutputRule, FnSourceRun, FutRunResult, RunResult, SourceTrait, StateTrait, + ZFContext, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_empty_state, +}; +use zenoh_flow_examples::RandomData; + +static SOURCE: &str = "Number"; + +#[derive(Serialize, Deserialize, Debug, ZFState)] +struct ExampleRandomSource {} + +impl ExampleRandomSource { + async fn run_1(_ctx: ZFContext) -> RunResult { + let mut results: HashMap> = HashMap::new(); + let d = RandomData { + d: rand::random::(), + }; + results.insert(String::from(SOURCE), zf_data!(d)); + async_std::task::sleep(std::time::Duration::from_secs(1)).await; + Ok(results) + } +} + +impl SourceTrait for ExampleRandomSource { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun { + Box::new(|ctx: ZFContext| -> FutRunResult { Box::pin(Self::run_1(ctx)) }) + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(zenoh_flow::default_output_rule) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +// //Also generated by macro +zenoh_flow::export_source!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + Ok(Box::new(ExampleRandomSource {}) as Box) +} diff --git a/zenoh-flow-examples/examples/sum-and-send.rs b/zenoh-flow-examples/examples/sum-and-send.rs new file mode 100644 index 00000000..9fc64fe0 --- /dev/null +++ b/zenoh-flow-examples/examples/sum-and-send.rs @@ -0,0 +1,127 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use zenoh_flow::runtime::message::ZFMessage; +use zenoh_flow::types::{ + DataTrait, FnInputRule, FnOutputRule, FnRun, InputRuleResult, OperatorTrait, OutputRuleResult, + RunResult, StateTrait, Token, ZFContext, ZFError, ZFInput, ZFLinkId, ZFResult, +}; +use zenoh_flow::zenoh_flow_derive::ZFState; +use zenoh_flow::{downcast_mut, get_input, zf_data}; +use zenoh_flow_examples::RandomData; + +use async_std::sync::Arc; + +#[derive(Serialize, Deserialize, Debug)] +struct SumAndSend { + pub state: SumAndSendState, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ZFState)] +struct SumAndSendState { + pub x: RandomData, +} + +static INPUT: &str = "Number"; +static OUTPUT: &str = "Number"; + +impl SumAndSend { + pub fn new() -> Self { + Self { + state: SumAndSendState { + x: RandomData { d: 0 }, + }, + } + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub fn run_1(ctx: ZFContext, mut inputs: ZFInput) -> RunResult { + let mut results: HashMap> = HashMap::new(); + + let mut guard = ctx.lock(); //getting the context + let mut _state = downcast_mut!(SumAndSendState, guard.state).unwrap(); //getting and downcasting state to right type + + let data = get_input!(RandomData, String::from(INPUT), inputs)?; + + let res = _state.x.d + data.d; + let res = RandomData { d: res }; + _state.x = res.clone(); + + results.insert(String::from(OUTPUT), zf_data!(res)); + Ok(results) + } + + pub fn or_1( + _ctx: ZFContext, + outputs: HashMap>, + ) -> OutputRuleResult { + let mut results = HashMap::new(); + for (k, v) in outputs { + // should be ZFMessage::from_data + results.insert(k, Arc::new(ZFMessage::from_data(v))); + } + Ok(results) + } +} + +impl OperatorTrait for SumAndSend { + fn get_input_rule(&self, ctx: ZFContext) -> Box { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(Self::ir_1), + _ => panic!("No way"), + } + } + + fn get_output_rule(&self, ctx: ZFContext) -> Box { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(Self::or_1), + _ => panic!("No way"), + } + } + + fn get_run(&self, ctx: ZFContext) -> Box { + let gctx = ctx.lock(); + match gctx.mode { + 0 => Box::new(Self::run_1), + _ => panic!("No way"), + } + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +// //Also generated by macro +zenoh_flow::export_operator!(register); + +extern "C" fn register( + configuration: Option>, +) -> ZFResult> { + Ok(Box::new(SumAndSend::new()) as Box) +} diff --git a/zenoh-flow-examples/examples/video-pipeline.rs b/zenoh-flow-examples/examples/video-pipeline.rs new file mode 100644 index 00000000..64709dfa --- /dev/null +++ b/zenoh-flow-examples/examples/video-pipeline.rs @@ -0,0 +1,270 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::{Arc, Mutex}; +use std::collections::HashMap; +use zenoh_flow::model::link::{ZFFromEndpoint, ZFToEndpoint}; +use zenoh_flow::{ + downcast, downcast_mut, get_input, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnInputRule, FnOutputRule, FnSinkRun, FnSourceRun, FutRunResult, FutSinkResult, + InputRuleResult, RunResult, SinkTrait, SourceTrait, StateTrait, Token, ZFContext, ZFError, + ZFInput, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_data, zf_spin_lock, +}; +use zenoh_flow_examples::ZFBytes; + +use opencv::{core, highgui, prelude::*, videoio}; + +static SOURCE: &str = "Frame"; +static INPUT: &str = "Frame"; + +#[derive(Debug)] +struct CameraSource { + pub state: CameraState, +} + +#[derive(Clone)] +struct InnerCameraAccess { + pub camera: Arc>, + pub encode_options: Arc>, +} + +#[derive(Serialize, Deserialize, ZFState, Clone)] +struct CameraState { + #[serde(skip_serializing, skip_deserializing)] + pub inner: Option, + + pub resolution: (i32, i32), + pub delay: u64, +} + +// because of opencv +impl std::fmt::Debug for CameraState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "CameraState: resolution:{:?} delay:{:?}", + self.resolution, self.delay + ) + } +} + +impl CameraSource { + fn new() -> Self { + let mut camera = videoio::VideoCapture::new(0, videoio::CAP_ANY).unwrap(); // 0 is the default camera + let opened = videoio::VideoCapture::is_opened(&camera).unwrap(); + if !opened { + panic!("Unable to open default camera!"); + } + let mut encode_options = opencv::types::VectorOfi32::new(); + encode_options.push(opencv::imgcodecs::IMWRITE_JPEG_QUALITY); + encode_options.push(90); + let inner = InnerCameraAccess { + camera: Arc::new(Mutex::new(camera)), + encode_options: Arc::new(Mutex::new(encode_options)), + }; + + let state = CameraState { + inner: Some(inner), + resolution: (800, 600), + delay: 40, + }; + + Self { state } + } + + async fn run_1(ctx: ZFContext) -> RunResult { + let mut results: HashMap> = HashMap::new(); + + let mut guard = ctx.async_lock().await; + let mut _state = downcast_mut!(CameraState, guard.state).unwrap(); //downcasting to right type + + let inner = _state.inner.as_ref().unwrap(); + + let mut cam = zf_spin_lock!(inner.camera); + let encode_options = zf_spin_lock!(inner.encode_options); + + let mut frame = core::Mat::default(); + cam.read(&mut frame).unwrap(); + + let mut reduced = Mat::default(); + opencv::imgproc::resize( + &frame, + &mut reduced, + opencv::core::Size::new(_state.resolution.0, _state.resolution.0), + 0.0, + 0.0, + opencv::imgproc::INTER_LINEAR, + ) + .unwrap(); + + let mut buf = opencv::types::VectorOfu8::new(); + opencv::imgcodecs::imencode(".jpeg", &reduced, &mut buf, &encode_options).unwrap(); + + let data = ZFBytes { + bytes: buf.to_vec(), + }; + + results.insert(String::from(SOURCE), zf_data!(data)); + + async_std::task::sleep(std::time::Duration::from_millis(_state.delay)); + + drop(cam); + drop(encode_options); + + Ok(results) + } +} + +impl SourceTrait for CameraSource { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun { + Box::new(|ctx: ZFContext| -> FutRunResult { Box::pin(Self::run_1(ctx)) }) + } + + fn get_output_rule(&self, _ctx: ZFContext) -> Box { + Box::new(zenoh_flow::default_output_rule) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +#[derive(Debug)] +struct VideoSink { + pub state: VideoState, +} + +#[derive(Serialize, Deserialize, ZFState, Clone, Debug)] +struct VideoState { + pub window_name: String, +} + +impl VideoSink { + pub fn new() -> Self { + let window_name = &format!("Video-Sink"); + highgui::named_window(window_name, 1).unwrap(); + let state = VideoState { + window_name: window_name.to_string(), + }; + Self { state } + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub async fn run_1(ctx: ZFContext, mut inputs: ZFInput) -> ZFResult<()> { + let guard = ctx.async_lock().await; //getting state, + let _state = downcast!(VideoState, guard.state).unwrap(); //downcasting to right type + + let data = get_input!(ZFBytes, String::from(INPUT), inputs).unwrap(); + + let decoded = opencv::imgcodecs::imdecode( + &opencv::types::VectorOfu8::from_iter(data.bytes.clone()), + opencv::imgcodecs::IMREAD_COLOR, + ) + .unwrap(); + + if decoded.size().unwrap().width > 0 { + highgui::imshow(&_state.window_name, &decoded).unwrap(); + } + + highgui::wait_key(10).unwrap(); + Ok(()) + } +} + +impl SinkTrait for VideoSink { + fn get_input_rule(&self, ctx: ZFContext) -> Box { + Box::new(Self::ir_1) + } + + fn get_run(&self, ctx: ZFContext) -> FnSinkRun { + Box::new(|ctx: ZFContext, inputs: ZFInput| -> FutSinkResult { + Box::pin(Self::run_1(ctx, inputs)) + }) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +#[async_std::main] +async fn main() { + let mut zf_graph = zenoh_flow::runtime::graph::DataFlowGraph::new(); + + let source = Box::new(CameraSource::new()); + let sink = Box::new(VideoSink::new()); + + zf_graph + .add_static_source( + "camera-source".to_string(), + // "camera".to_string(), + String::from(SOURCE), + source, + None, + ) + .unwrap(); + + zf_graph + .add_static_sink( + "video-sink".to_string(), + // "window".to_string(), + String::from(INPUT), + sink, + None, + ) + .unwrap(); + + zf_graph + .add_link( + ZFFromEndpoint { + id: "camera-source".to_string(), + output: String::from(SOURCE), + }, + ZFToEndpoint { + id: "video-sink".to_string(), + input: String::from(INPUT), + }, + None, + None, + None, + ) + .unwrap(); + + zf_graph.make_connections("local").await; + + let runners = zf_graph.get_runners(); + for runner in runners { + async_std::task::spawn(async move { + let mut runner = runner.lock().await; + runner.run().await.unwrap(); + }); + } + + let () = std::future::pending().await; +} diff --git a/zenoh-flow-examples/examples/video-sink.rs b/zenoh-flow-examples/examples/video-sink.rs new file mode 100644 index 00000000..76b6727c --- /dev/null +++ b/zenoh-flow-examples/examples/video-sink.rs @@ -0,0 +1,110 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use zenoh_flow::{ + downcast, downcast_mut, get_input, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnInputRule, FnSinkRun, FutSinkResult, InputRuleResult, SinkTrait, StateTrait, + Token, ZFContext, ZFError, ZFInput, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, + zf_empty_state, zf_spin_lock, +}; +use zenoh_flow_examples::{ZFBytes, ZFOpenCVBytes}; + +use opencv::{highgui, prelude::*}; + +static INPUT: &str = "Frame"; + +#[derive(Debug)] +struct VideoSink {} + +#[derive(Serialize, Deserialize, ZFState, Clone, Debug)] +struct VideoState { + pub window_name: String, +} + +impl VideoSink { + pub fn new() -> Self { + let window_name = &format!("Video-Sink"); + highgui::named_window(window_name, 1).unwrap(); + Self {} + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub async fn run_1(_ctx: ZFContext, mut inputs: ZFInput) -> ZFResult<()> { + let mut results: HashMap> = HashMap::new(); + + let window_name = &format!("Video-Sink"); + + let data = get_input!(ZFBytes, String::from(INPUT), inputs).unwrap(); + + let decoded = match opencv::imgcodecs::imdecode( + &opencv::types::VectorOfu8::from_iter(data.bytes.clone()), + opencv::imgcodecs::IMREAD_COLOR, + ) { + Ok(d) => d, + Err(e) => panic!("Unable to decode {:?}", e), + }; + + if decoded.size().unwrap().width > 0 { + match highgui::imshow(window_name, &decoded) { + Ok(_) => (), + Err(e) => eprintln!("Error when display {:?}", e), + }; + } + + highgui::wait_key(10); + + Ok(()) + } +} + +impl SinkTrait for VideoSink { + fn get_input_rule(&self, _ctx: ZFContext) -> Box { + Box::new(Self::ir_1) + } + + fn get_run(&self, ctx: ZFContext) -> FnSinkRun { + Box::new(|ctx: ZFContext, inputs: ZFInput| -> FutSinkResult { + Box::pin(Self::run_1(ctx, inputs)) + }) + } + + fn get_state(&self) -> Box { + zf_empty_state!() + } +} + +// //Also generated by macro +zenoh_flow::export_sink!(register); + +extern "C" fn register( + _configuration: Option>, +) -> ZFResult> { + Ok(Box::new(VideoSink::new()) as Box) +} diff --git a/zenoh-flow-examples/examples/zenoh-sink.rs b/zenoh-flow-examples/examples/zenoh-sink.rs new file mode 100644 index 00000000..161e970b --- /dev/null +++ b/zenoh-flow-examples/examples/zenoh-sink.rs @@ -0,0 +1,113 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; +use std::collections::HashMap; +use zenoh_flow::{ + downcast, get_input, + serde::{Deserialize, Serialize}, + types::{ + DataTrait, FnInputRule, FnSinkRun, FutSinkResult, InputRuleResult, SinkTrait, StateTrait, + Token, ZFContext, ZFError, ZFInput, ZFLinkId, ZFResult, + }, + zenoh_flow_derive::ZFState, +}; + +use zenoh_flow_examples::ZFBytes; + +use zenoh::net::config; +use zenoh::net::{open, Session}; +use zenoh::ZFuture; + +static INPUT: &str = "Frame"; + +#[derive(Debug)] +struct ExampleGenericZenohSink { + state: ZSinkState, +} + +#[derive(Debug, Clone)] +struct ZSinkInner { + session: Arc, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ZFState)] +struct ZSinkState { + #[serde(skip_serializing, skip_deserializing)] + inner: Option, +} + +impl ExampleGenericZenohSink { + pub fn new() -> Self { + Self { + state: ZSinkState { + inner: Some(ZSinkInner { + session: Arc::new(open(config::peer()).wait().unwrap()), + }), + }, + } + } + + pub fn ir_1(_ctx: ZFContext, inputs: &mut HashMap) -> InputRuleResult { + if let Some(token) = inputs.get(INPUT) { + match token { + Token::Ready(_) => Ok(true), + Token::NotReady(_) => Ok(false), + } + } else { + Err(ZFError::MissingInput(String::from(INPUT))) + } + } + + pub async fn run_1(ctx: ZFContext, mut inputs: ZFInput) -> ZFResult<()> { + let guard = ctx.async_lock().await; //getting state, + let _state = downcast!(ZSinkState, guard.state).unwrap(); //downcasting to right type + + let inner = _state.inner.as_ref().unwrap(); + + let path = format!("/zf/probe/{}", String::from(INPUT)); + let data = get_input!(ZFBytes, String::from(INPUT), inputs).unwrap(); + + inner + .session + .write(&path.into(), data.bytes.clone().into()) + .wait() + .unwrap(); + Ok(()) + } +} + +impl SinkTrait for ExampleGenericZenohSink { + fn get_input_rule(&self, _ctx: ZFContext) -> Box { + Box::new(Self::ir_1) + } + + fn get_run(&self, ctx: ZFContext) -> FnSinkRun { + Box::new(|ctx: ZFContext, inputs: ZFInput| -> FutSinkResult { + Box::pin(Self::run_1(ctx, inputs)) + }) + } + + fn get_state(&self) -> Box { + Box::new(self.state.clone()) + } +} + +zenoh_flow::export_sink!(register); + +extern "C" fn register( + _configuration: Option>, +) -> ZFResult> { + Ok(Box::new(ExampleGenericZenohSink::new()) as Box) +} diff --git a/zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml b/zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml new file mode 100644 index 00000000..a552037f --- /dev/null +++ b/zenoh-flow-examples/graphs/dnn-object-detection-multi-runtime.yaml @@ -0,0 +1,44 @@ +flow: DNNPipeline # it should have a id, from this we will have an Instance of the graph, UUID => flow +operators: + - id : ObjectDetection # this should be unique in the graph + uri: file://./target/release/examples/libobject_detection_dnn.so + inputs: [Frame] + outputs: [Frame] #port id = String + configuration: + neural-network: /home/gabri/Workspace/experiments/yolov3-face.cfg + network-weights: /home/gabri/Workspace/experiments/yolov3-wider_16000.weights + network-classes: /home/gabri/Workspace/experiments/face_classes.txt +sources: # sources can have only ONE output + - id : Camera0 + uri: file://./target/release/examples/libcamera_source.so + output: Frame + configuration: + camera: /dev/video0 + resolution: 800x600 + fps: 24 +sinks: # sources can have only ONE input + - id : Window + uri: file://./target/release/examples/libvideo_sink.so + input: Frame +links: +- from: + id : Camera0 + output : Frame # Output + to: + id : ObjectDetection + input : Frame # Input +- from: + id : ObjectDetection + output : Frame + to: + id : Window + input : Frame + + +mapping: + - id: Window + runtime: foo + - id: Camera0 + runtime: bar + - id: ObjectDetection + runtime: cuda \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/dnn-object-detection.yaml b/zenoh-flow-examples/graphs/dnn-object-detection.yaml new file mode 100644 index 00000000..17ded331 --- /dev/null +++ b/zenoh-flow-examples/graphs/dnn-object-detection.yaml @@ -0,0 +1,35 @@ +flow: DNNPipeline # it should have a id, from this we will have an Instance of the graph, UUID => flow +operators: + - id : ObjectDetection # this should be unique in the graph + uri: file://./target/release/examples/libobject_detection_dnn.so + inputs: [Frame] + outputs: [Frame] #port id = String + configuration: + neural-network: /home/gabri/Workspace/experiments/yolov3-face.cfg + network-weights: /home/gabri/Workspace/experiments/yolov3-wider_16000.weights + network-classes: /home/gabri/Workspace/experiments/face_classes.txt +sources: # sources can have only ONE output + - id : Camera0 + uri: file://./target/release/examples/libcamera_source.so + output: Frame + configuration: + camera: /dev/video0 + resolution: 800x600 + fps: 24 +sinks: # sources can have only ONE input + - id : Window + uri: file://./target/release/examples/libvideo_sink.so + input: Frame +links: +- from: + id : Camera0 + output : Frame # Output + to: + id : ObjectDetection + input : Frame # Input +- from: + id : ObjectDetection + output : Frame + to: + id : Window + input : Frame \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml b/zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml new file mode 100644 index 00000000..fcece14c --- /dev/null +++ b/zenoh-flow-examples/graphs/face-detection-multi-runtime.yaml @@ -0,0 +1,44 @@ +flow: FacePipeline # it should have a id, from this we will have an Instance of the graph, UUID => flow +operators: + - id : FaceDetection # this should be unique in the graph + uri: file://./target/release/examples/libface_detection.so + inputs: [Frame] + outputs: [Frame] #port id = String + # configuration: + # neural-network: haarcascades/haarcascade_frontalface_alt_tree.xml + # neural-network: haarcascades/haarcascade_eye.xml #eyes only + # more configurations can be found in opencv: /usr/share/opencv4/haarcascades +sources: # sources can have only ONE output + - id : Camera0 + uri: file://./target/release/examples/libcamera_source.so + output: Frame + configuration: + camera: /dev/video0 + resolution: 800x600 + fps: 24 +sinks: # sources can have only ONE input + - id : Window + uri: file://./target/release/examples/libvideo_sink.so + input: Frame +links: +- from: + id : Camera0 + output : Frame # Output + to: + id : FaceDetection + input : Frame # Input +- from: + id : FaceDetection + output : Frame + to: + id : Window + input : Frame + + +mapping: + - id: Window + runtime: gigot + - id: Camera0 + runtime: nuc + - id: FaceDetection + runtime: leia \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/face_detection.yaml b/zenoh-flow-examples/graphs/face_detection.yaml new file mode 100644 index 00000000..fb1a9958 --- /dev/null +++ b/zenoh-flow-examples/graphs/face_detection.yaml @@ -0,0 +1,35 @@ +flow: FacePipeline # it should have a name, from this we will have an Instance of the graph, UUID => flow +operators: + - id : FaceDetection # this should be unique in the graph... + uri: file://./target/release/examples/libface_detection.so + inputs: [Frame] + outputs: [Frame] #port id = String + # configuration: + # neural-network: haarcascades/haarcascade_frontalface_alt_tree.xml + # neural-network: haarcascades/haarcascade_eye.xml #eyes only + # more configurations can be found in opencv: /usr/share/opencv4/haarcascades +sources: # sources can have only ONE output + - id : Camera0 + uri: file://./target/release/examples/libcamera_source.so + output: Frame + configuration: + camera: /dev/video0 + resolution: 800x600 + fps: 24 +sinks: # sources can have only ONE input + - id : Window + uri: file://./target/release/examples/libvideo_sink.so + input: Frame +links: +- from: + id : Camera0 + output : Frame # Output + to: + id : FaceDetection + input : Frame # Input +- from: + id : FaceDetection + output : Frame + to: + id : Window + input : Frame \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/fizz-buzz-multiple-runtimes.yaml b/zenoh-flow-examples/graphs/fizz-buzz-multiple-runtimes.yaml new file mode 100644 index 00000000..54eedd7e --- /dev/null +++ b/zenoh-flow-examples/graphs/fizz-buzz-multiple-runtimes.yaml @@ -0,0 +1,62 @@ +flow: FizzBuzz + +operators: + - id: FizzOperator + uri: file://./target/debug/examples/libexample_fizz.dylib + inputs: [Int] + outputs: [Int, Str] + + - id: BuzzOperator + uri: file://./target/debug/examples/libexample_buzz.dylib + inputs: [Int, Str] + outputs: [Str] + +sources: + - id: ManualSenderOperator + uri: file://./target/debug/examples/libmanual_source.dylib + output: Int + +sinks: + - id: ReceiverOperator + uri: file://./target/debug/examples/libgeneric_sink.dylib + input: Str + +links: + - from: + id: ManualSenderOperator + output: Int + to: + id: FizzOperator + input: Int + + - from: + id: FizzOperator + output: Int + to: + id: BuzzOperator + input: Int + + - from: + id: FizzOperator + output: Str + to: + id: BuzzOperator + input: Str + + - from: + id: BuzzOperator + output: Str + to: + id: ReceiverOperator + input: Str + + +mapping: + - id: FizzOperator + runtime: foo + - id: BuzzOperator + runtime: bar + - id: ManualSenderOperator + runtime: foo + - id: ReceiverOperator + runtime: bar \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/fizz_buzz_pipeline.yaml b/zenoh-flow-examples/graphs/fizz_buzz_pipeline.yaml new file mode 100644 index 00000000..222365f5 --- /dev/null +++ b/zenoh-flow-examples/graphs/fizz_buzz_pipeline.yaml @@ -0,0 +1,50 @@ +flow: FizzBuzz + +operators: + - id: FizzOperator + uri: file://./target/debug/examples/libexample_fizz.dylib + inputs: [Int] + outputs: [Int, Str] + - id: BuzzOperator + uri: file://./target/debug/examples/libexample_buzz.dylib + inputs: [Int, Str] + outputs: [Str] + +sources: + - id: ManualSenderOperator + uri: file://./target/debug/examples/libmanual_source.dylib + output: Int + +sinks: + - id: ReceiverOperator + uri: file://./target/debug/examples/libgeneric_sink.dylib + input: Str + +links: + - from: + id: ManualSenderOperator + output: Int + to: + id: FizzOperator + input: Int + + - from: + id: FizzOperator + output: Int + to: + id: BuzzOperator + input: Int + + - from: + id: FizzOperator + output: Str + to: + id: BuzzOperator + input: Str + + - from: + id: BuzzOperator + output: Str + to: + id: ReceiverOperator + input: Str \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/random_source.yaml b/zenoh-flow-examples/graphs/random_source.yaml new file mode 100644 index 00000000..2cd5ed95 --- /dev/null +++ b/zenoh-flow-examples/graphs/random_source.yaml @@ -0,0 +1,17 @@ +flow: RandomGeneration +operators: [] +sources: + - id : RandomGenerator + uri: file://./target/debug/examples/librandom_source.dylib + output: Number +sinks: + - id : PrintSink + uri: file://./target/debug/examples/libgeneric_sink.dylib + input: Number +links: +- from: + id : RandomGenerator + output : Number + to: + id : PrintSink + input : Number \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/simple_pipeline.yaml b/zenoh-flow-examples/graphs/simple_pipeline.yaml new file mode 100644 index 00000000..855a2def --- /dev/null +++ b/zenoh-flow-examples/graphs/simple_pipeline.yaml @@ -0,0 +1,27 @@ +flow: SimplePipeline +operators: + - id : SumOperator + uri: file://./target/debug/examples/libsum_and_send.dylib + inputs: [Number] + outputs: [Number] +sources: + - id : Counter + uri: file://./target/debug/examples/libcounter_source.dylib + output: Number +sinks: + - id : PrintSink + uri: file://./target/debug/examples/libgeneric_sink.dylib + input: Number +links: +- from: + id : Counter + output : Number + to: + id : SumOperator + input : Number +- from: + id : SumOperator + output : Number + to: + id : PrintSink + input : Number \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/video_pipeline.yaml b/zenoh-flow-examples/graphs/video_pipeline.yaml new file mode 100644 index 00000000..de757678 --- /dev/null +++ b/zenoh-flow-examples/graphs/video_pipeline.yaml @@ -0,0 +1,17 @@ +flow: VideoPipeline +operators: [] +sources: + - id : Camera + uri: file://./target/debug/examples/libcamera_source.so + output: Frame +sinks: + - id : Window + uri: file://./target/debug/examples/libvideo_sink.so + input: Frame +links: +- from: + id : Camera + output : Frame + to: + id : Window + input : Frame \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/zface_detection.yaml b/zenoh-flow-examples/graphs/zface_detection.yaml new file mode 100644 index 00000000..ac2a84ff --- /dev/null +++ b/zenoh-flow-examples/graphs/zface_detection.yaml @@ -0,0 +1,27 @@ +flow: zface-detection +operators: + - id : FaceDetection + uri: file://./target/debug/examples/libface_detection.dylib + inputs: [Frame] + outputs: [Frame] +sources: + - id : Camera0 + uri: file://./target/debug/examples/libcamera_source.dylib + output: Frame +sinks: + - id : ZSink + uri: file://./target/debug/examples/libzenoh_sink.dylib + input: Frame +links: +- from: + id : Camera0 + output : Frame + to: + id : FaceDetection + input : Frame +- from: + id : FaceDetection + output : Frame + to: + id : ZSink + input : Frame \ No newline at end of file diff --git a/zenoh-flow-examples/graphs/zvideo_pipeline.yaml b/zenoh-flow-examples/graphs/zvideo_pipeline.yaml new file mode 100644 index 00000000..65f6d9f0 --- /dev/null +++ b/zenoh-flow-examples/graphs/zvideo_pipeline.yaml @@ -0,0 +1,17 @@ +flow: ZVideoPipeline +operators: [] +sources: + - id : Camera + uri: file://./target/debug/examples/libcamera_source.dylib + output: Frame +sinks: + - id : ZSink + uri: file://./target/debug/examples/libzenoh_sink.dylib + input: Frame +links: +- from: + id : Camera + output : Frame + to: + id : ZSink + input : Frame \ No newline at end of file diff --git a/zenoh-flow-examples/src/lib.rs b/zenoh-flow-examples/src/lib.rs new file mode 100644 index 00000000..149f0ec3 --- /dev/null +++ b/zenoh-flow-examples/src/lib.rs @@ -0,0 +1,47 @@ +use async_std::sync::Mutex; +use std::cell::RefCell; +use zenoh_flow::serde::{Deserialize, Serialize}; +use zenoh_flow::zenoh_flow_derive::{ZFData, ZFState}; +// We may want to provide some "built-in" types + +#[derive(Debug, Clone, Serialize, Deserialize, ZFData)] +pub struct ZFString(pub String); + +impl From for ZFString { + fn from(s: String) -> Self { + ZFString(s) + } +} + +impl From<&str> for ZFString { + fn from(s: &str) -> Self { + ZFString(s.to_owned()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ZFData)] +pub struct ZFUsize(pub usize); + +#[derive(Debug, Clone, Serialize, Deserialize, ZFState)] +pub struct ZFEmptyState; + +#[derive(Serialize, Deserialize, Debug, Clone, ZFData)] +pub struct RandomData { + pub d: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, ZFData)] +pub struct ZFBytes { + pub bytes: Vec, +} + +#[derive(Debug, Serialize, Deserialize, ZFData)] +pub struct ZFOpenCVBytes { + #[serde(skip_serializing, skip_deserializing)] + pub bytes: Mutex>, +} + +// #[derive(Debug, ZFData)] +// pub struct OpenCVMat { +// pub mat: Mutex>, +// } diff --git a/zenoh-flow/Cargo.toml b/zenoh-flow/Cargo.toml new file mode 100644 index 00000000..0ffb4ea8 --- /dev/null +++ b/zenoh-flow/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "zenoh-flow" +version = "0.1.0" +repository = "https://github.com/eclipse-zenoh/zenoh-flow" +homepage = "http://zenoh.io" +authors = ["kydos ", + "gabrik ", + "Julien Loudet ",] +edition = "2018" +license = " EPL-2.0 OR Apache-2.0" +categories = ["network-programming"] +description = "Zenoh-Flow: zenoh-based data-flow programming framework for computations that span from the cloud to the device." +readme = "README.md" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +env_logger = "0.9" +serde_derive = "1.0.55" +serde = { version = "1.0.55", features = ["derive", "rc"] } +serde_yaml = {version = "0.8.13"} +serde_json = "1.0" +derive_more = "0.99.10" +petgraph = "0.6.0" +structopt = "0.3.21" +libloading = "0.7.0" +zenoh-flow-derive = {path = "../zenoh-flow-derive"} +log = "0.4" +zenoh = { git = "https://github.com/eclipse-zenoh/zenoh.git", branch = "master", default-features = false, features = ["transport_tcp","transport_udp"] } +zenoh-util = { git = "https://github.com/eclipse-zenoh/zenoh.git", branch = "master" } +url = "2.2.2" +uuid = { version = "0.8", features = ["serde", "v4"] } +futures = "0.3.15" +async-std = { version = "=1.9.0", features = ["attributes"] } +async-trait = "0.1.50" +event-listener = "2.5.1" +paste = "1.0" +bincode = "1.3.3" +flume = "0.10" +typetag = "0.1" + +[build-dependencies] +rustc_version = "0.4.0" + +[[bin]] +name = "runtime" diff --git a/zenoh-flow/build.rs b/zenoh-flow/build.rs new file mode 100644 index 00000000..189a17ec --- /dev/null +++ b/zenoh-flow/build.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +fn main() { + let version = rustc_version::version().unwrap(); + println!("cargo:rustc-env=RUSTC_VERSION={}", version); +} diff --git a/zenoh-flow/src/bin/runtime.rs b/zenoh-flow/src/bin/runtime.rs new file mode 100644 index 00000000..b49667bc --- /dev/null +++ b/zenoh-flow/src/bin/runtime.rs @@ -0,0 +1,115 @@ +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +// TODO: this should become a deamon. + +use std::fs::{File, *}; +use std::io::Write; +use std::path::Path; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "dpn")] +struct Opt { + #[structopt(short = "g", long = "graph-file")] + graph_file: String, + #[structopt(short = "o", long = "out-file", default_value = "output.dot")] + outfile: String, + #[structopt(short = "r", long = "runtime")] + runtime: String, +} + +fn _write_record_to_file(record: zenoh_flow::model::dataflow::DataFlowRecord, filename: &str) { + let path = Path::new(filename); + let mut write_file = File::create(path).unwrap(); + write!(write_file, "{}", record.to_yaml().unwrap()).unwrap(); +} + +#[async_std::main] +async fn main() { + env_logger::init(); + + let opt = Opt::from_args(); + let yaml_df = read_to_string(opt.graph_file).unwrap(); + + // loading the descriptor + let df = zenoh_flow::model::dataflow::DataFlowDescriptor::from_yaml(yaml_df).unwrap(); + + // mapping to infrastructure + let mapped = zenoh_flow::runtime::map_to_infrastructure(df, &opt.runtime) + .await + .unwrap(); + + // creating record + let dfr = + zenoh_flow::model::dataflow::DataFlowRecord::from_dataflow_descriptor(mapped).unwrap(); + + //write_record_to_file(dfr.clone(), "computed-record.yaml"); + + // creating graph + let mut dataflow_graph = + zenoh_flow::runtime::graph::DataFlowGraph::from_dataflow_record(dfr).unwrap(); + + let dot_notation = dataflow_graph.to_dot_notation(); + + let mut file = File::create(opt.outfile).unwrap(); + write!(file, "{}", dot_notation).unwrap(); + file.sync_all().unwrap(); + + // instantiating + dataflow_graph.load(&opt.runtime).unwrap(); + + dataflow_graph.make_connections(&opt.runtime).await.unwrap(); + + // let mut runners = dataflow_graph.get_sinks(); + // for runner in runners.drain(..) { + // async_std::task::spawn(async move { + // let mut runner = runner.lock().await; + // runner.run().await.unwrap(); + // }); + // } + + let mut sinks = dataflow_graph.get_sinks(); + for runner in sinks.drain(..) { + async_std::task::spawn(async move { + let mut runner = runner.lock().await; + runner.run().await.unwrap(); + }); + } + + let mut operators = dataflow_graph.get_operators(); + for runner in operators.drain(..) { + async_std::task::spawn(async move { + let mut runner = runner.lock().await; + runner.run().await.unwrap(); + }); + } + + let mut connectors = dataflow_graph.get_connectors(); + for runner in connectors.drain(..) { + async_std::task::spawn(async move { + let mut runner = runner.lock().await; + runner.run().await.unwrap(); + }); + } + + let mut sources = dataflow_graph.get_sources(); + for runner in sources.drain(..) { + async_std::task::spawn(async move { + let mut runner = runner.lock().await; + runner.run().await.unwrap(); + }); + } + + let () = std::future::pending().await; +} diff --git a/zenoh-flow/src/lib.rs b/zenoh-flow/src/lib.rs new file mode 100644 index 00000000..cc6b2f4c --- /dev/null +++ b/zenoh-flow/src/lib.rs @@ -0,0 +1,29 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +pub use ::zenoh_flow_derive; + +pub use ::async_std; +pub use ::bincode; +pub use ::paste; +pub use ::serde; +pub use ::typetag; + +pub mod model; +pub mod runtime; +pub mod types; +pub use types::*; + +pub mod macros; +pub use macros::*; diff --git a/zenoh-flow/src/macros.rs b/zenoh-flow/src/macros.rs new file mode 100644 index 00000000..53ad185c --- /dev/null +++ b/zenoh-flow/src/macros.rs @@ -0,0 +1,161 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +#[macro_export] +macro_rules! export_operator { + ($register:expr) => { + #[doc(hidden)] + #[no_mangle] + pub static zfoperator_declaration: $crate::runtime::runner::ZFOperatorDeclaration = + $crate::runtime::runner::ZFOperatorDeclaration { + rustc_version: $crate::runtime::loader::RUSTC_VERSION, + core_version: $crate::runtime::loader::CORE_VERSION, + register: $register, + }; + }; +} + +#[macro_export] +macro_rules! export_source { + ($register:expr) => { + #[doc(hidden)] + #[no_mangle] + pub static zfsource_declaration: $crate::runtime::runner::ZFSourceDeclaration = + $crate::runtime::runner::ZFSourceDeclaration { + rustc_version: $crate::runtime::loader::RUSTC_VERSION, + core_version: $crate::runtime::loader::CORE_VERSION, + register: $register, + }; + }; +} + +#[macro_export] +macro_rules! export_sink { + ($register:expr) => { + #[doc(hidden)] + #[no_mangle] + pub static zfsink_declaration: $crate::runtime::runner::ZFSinkDeclaration = + $crate::runtime::runner::ZFSinkDeclaration { + rustc_version: $crate::runtime::loader::RUSTC_VERSION, + core_version: $crate::runtime::loader::CORE_VERSION, + register: $register, + }; + }; +} + +#[macro_export] +macro_rules! zf_spin_lock { + ($val : expr) => { + loop { + match $val.try_lock() { + Some(x) => break x, + None => std::hint::spin_loop(), + } + } + }; +} + +#[macro_export] +macro_rules! zf_data { + ($val : expr) => { + zenoh_flow::async_std::sync::Arc::new($val) + }; +} + +#[macro_export] +macro_rules! downcast { + ($ident : ident, $val : expr) => { + $val.as_any().downcast_ref::<$ident>() + }; +} + +#[macro_export] +macro_rules! downcast_mut { + ($ident : ident, $val : expr) => { + $val.as_mut_any().downcast_mut::<$ident>() + }; +} + +#[macro_export] +macro_rules! take_state { + ($ident : ident, $ctx : expr) => { + match $ctx.take_state() { + Some(mut state) => match zenoh_flow::downcast_mut!($ident, state) { + Some(mut data) => Ok((state, data)), + None => Err(zenoh_flow::types::ZFError::InvalidState), + }, + None => Err(zenoh_flow::types::ZFError::MissingState), + } + }; +} + +#[macro_export] +macro_rules! get_state { + ($ident : ident, $ctx : expr) => { + match $ctx.get_state() { + Some(state) => match zenoh_flow::downcast!($ident, state) { + Some(data) => Ok((state, data)), + None => Err(zenoh_flow::types::ZFError::InvalidState), + }, + None => Err(zenoh_flow::types::ZFError::MissingState), + } + }; +} + +#[macro_export] +macro_rules! get_input { + ($ident : ident, $index : expr, $map : expr) => { + match $map.get_mut(&$index) { + Some(mut d) => match d { + zenoh_flow::types::ZFData::Deserialized(de) => { + match zenoh_flow::downcast!($ident, de) { + Some(data) => Ok(data), + None => Err(zenoh_flow::types::ZFError::InvalidData($index)), + } + } + zenoh_flow::types::ZFData::Serialized(ser) => { + let de: Arc = bincode::deserialize(&ser) + .map_err(|_| zenoh_flow::types::ZFError::DeseralizationError)?; + + *d = zenoh_flow::types::ZFData::Deserialized(de); + + match d { + zenoh_flow::types::ZFData::Deserialized(de) => { + match zenoh_flow::downcast!($ident, de) { + Some(data) => Ok(data), + None => Err(zenoh_flow::types::ZFError::InvalidData($index)), + } + } + _ => Err(zenoh_flow::types::ZFError::Unimplemented), + } + } + }, + None => Err(zenoh_flow::types::ZFError::MissingInput($index)), + } + }; +} + +#[macro_export] +macro_rules! zf_empty_state { + () => { + Box::new(zenoh_flow::EmptyState {}) + }; +} + +#[macro_export] +macro_rules! zf_source_result { + ($result : expr) => { + Box::pin($result) + }; +} diff --git a/zenoh-flow/src/model/connector.rs b/zenoh-flow/src/model/connector.rs new file mode 100644 index 00000000..7b5c6a24 --- /dev/null +++ b/zenoh-flow/src/model/connector.rs @@ -0,0 +1,50 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::serde::{Deserialize, Serialize}; +use crate::{ZFLinkId, ZFRuntimeID}; + +#[derive(PartialEq, Serialize, Deserialize, Debug, Clone)] +pub enum ZFConnectorKind { + Sender, + Receiver, +} + +impl std::fmt::Display for ZFConnectorKind { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Receiver => write!(f, "Receiver"), + Self::Sender => write!(f, "Sender"), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFConnectorRecord { + pub kind: ZFConnectorKind, + pub id: String, + pub resource: String, + pub link_id: ZFLinkId, + pub runtime: ZFRuntimeID, +} + +impl std::fmt::Display for ZFConnectorRecord { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{} - {} - Kind: Connector{}", + self.id, self.resource, self.kind + ) + } +} diff --git a/zenoh-flow/src/model/dataflow.rs b/zenoh-flow/src/model/dataflow.rs new file mode 100644 index 00000000..68f1bf8e --- /dev/null +++ b/zenoh-flow/src/model/dataflow.rs @@ -0,0 +1,358 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::model::connector::{ZFConnectorKind, ZFConnectorRecord}; +use crate::model::link::{ZFFromEndpoint, ZFLinkDescriptor, ZFToEndpoint}; +use crate::model::operator::{ + ZFOperatorDescriptor, ZFOperatorRecord, ZFSinkDescriptor, ZFSinkRecord, ZFSourceDescriptor, + ZFSourceRecord, +}; +use crate::runtime::graph::node::DataFlowNode; +use crate::serde::{Deserialize, Serialize}; +use crate::types::{ZFError, ZFOperatorId, ZFResult, ZFRuntimeID}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DataFlowDescriptor { + pub flow: String, + pub operators: Vec, + pub sources: Vec, + pub sinks: Vec, + pub links: Vec, + pub mapping: Option>, +} + +impl DataFlowDescriptor { + pub fn from_yaml(data: String) -> ZFResult { + serde_yaml::from_str::(&data) + .map_err(|e| ZFError::ParsingError(format!("{}", e))) + } + + pub fn from_json(data: String) -> ZFResult { + serde_json::from_str::(&data) + .map_err(|e| ZFError::ParsingError(format!("{}", e))) + } + pub fn to_json(&self) -> ZFResult { + serde_json::to_string(&self).map_err(|_| ZFError::SerializationError) + } + + pub fn to_yaml(&self) -> ZFResult { + serde_yaml::to_string(&self).map_err(|_| ZFError::SerializationError) + } + + fn _get_operator(&self, id: ZFOperatorId) -> Option { + self.operators.iter().find(|&o| o.id == id).cloned() + } + + fn _get_source(&self, id: ZFOperatorId) -> Option { + self.sources.iter().find(|&o| o.id == id).cloned() + } + + fn _get_sink(&self, id: ZFOperatorId) -> Option { + self.sinks.iter().find(|&o| o.id == id).cloned() + } + + pub fn get_mapping(&self, id: &ZFOperatorId) -> Option { + match &self.mapping { + Some(mapping) => mapping + .iter() + .find(|&o| o.id == *id) + .map(|m| m.runtime.clone()), + None => None, + } + } + + pub fn add_mapping(&mut self, mapping: Mapping) { + match self.mapping.as_mut() { + Some(m) => m.push(mapping), + None => self.mapping = Some(vec![mapping]), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Mapping { + pub id: ZFOperatorId, + pub runtime: ZFRuntimeID, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DataFlowRecord { + pub uuid: Uuid, + pub flow: String, + pub operators: Vec, + pub sinks: Vec, + pub sources: Vec, + pub connectors: Vec, + pub links: Vec, +} + +impl DataFlowRecord { + pub fn from_yaml(data: String) -> ZFResult { + serde_yaml::from_str::(&data) + .map_err(|e| ZFError::ParsingError(format!("{}", e))) + } + + pub fn from_json(data: String) -> ZFResult { + serde_json::from_str::(&data) + .map_err(|e| ZFError::ParsingError(format!("{}", e))) + } + + pub fn to_json(&self) -> ZFResult { + serde_json::to_string(&self).map_err(|_| ZFError::SerializationError) + } + + pub fn to_yaml(&self) -> ZFResult { + serde_yaml::to_string(&self).map_err(|_| ZFError::SerializationError) + } + + pub fn find_component_runtime(&self, id: &ZFOperatorId) -> Option { + match self.get_operator(id) { + Some(o) => Some(o.runtime), + None => match self.get_source(id) { + Some(s) => Some(s.runtime), + None => match self.get_sink(id) { + Some(s) => Some(s.runtime), + None => None, + }, + }, + } + } + + pub fn find_node(&self, id: &ZFOperatorId) -> Option { + match self.get_operator(id) { + Some(o) => Some(DataFlowNode::Operator(o)), + None => match self.get_source(id) { + Some(s) => Some(DataFlowNode::Source(s)), + None => match self.get_sink(id) { + Some(s) => Some(DataFlowNode::Sink(s)), + None => self.get_connector(id).map(DataFlowNode::Connector), + }, + }, + } + } + + fn get_operator(&self, id: &ZFOperatorId) -> Option { + self.operators.iter().find(|&o| o.id == *id).cloned() + } + + fn get_source(&self, id: &ZFOperatorId) -> Option { + self.sources.iter().find(|&o| o.id == *id).cloned() + } + + fn get_sink(&self, id: &ZFOperatorId) -> Option { + self.sinks.iter().find(|&o| o.id == *id).cloned() + } + + fn get_connector(&self, id: &String) -> Option { + self.connectors.iter().find(|&o| o.id == *id).cloned() + } + + fn add_links(&mut self, links: &[ZFLinkDescriptor]) -> ZFResult<()> { + for l in links { + if l.from.output != l.to.input { + return Err(ZFError::PortIdNotMatching(( + l.from.output.clone(), + l.to.input.clone(), + ))); + } + + let from_runtime = match self.find_component_runtime(&l.from.id) { + Some(rt) => rt, + None => { + return Err(ZFError::Uncompleted(format!( + "Unable to find runtime for {}", + &l.from.id + ))) + } + }; + + let to_runtime = match self.find_component_runtime(&l.to.id) { + Some(rt) => rt, + None => { + return Err(ZFError::Uncompleted(format!( + "Unable to find runtime for {}", + &l.to.id + ))) + } + }; + + if from_runtime == to_runtime { + // link between components on the same runtime + self.links.push(l.clone()) + } else { + // link between component on different runtime + // here we have to create the connectors information + // and add the new links + + // creating zenoh resource name + let z_resource_name = format!( + "/zf/data/{}/{}/{}/{}", + &self.flow, &self.uuid, &l.from.id, &l.from.output + ); + + // We only create a sender if none was created for the same resource. The rationale + // is to avoid creating multiple publisher for the same resource in case an operator + // acts as a multiplexor. + if !self + .connectors + .iter() + .any(|c| c.kind == ZFConnectorKind::Sender && c.resource == z_resource_name) + { + // creating sender + let sender_id = format!( + "sender-{}-{}-{}-{}", + &self.flow, &self.uuid, &l.from.id, &l.from.output + ); + let sender = ZFConnectorRecord { + kind: ZFConnectorKind::Sender, + id: sender_id.clone(), + resource: z_resource_name.clone(), + link_id: l.from.output.clone(), + runtime: from_runtime, + }; + + // creating link between component and sender + let link_sender = ZFLinkDescriptor { + from: l.from.clone(), + to: ZFToEndpoint { + id: sender_id, + input: sender.link_id.clone(), + }, + size: None, + queueing_policy: None, + priority: None, + }; + + // storing info in the dataflow record + self.connectors.push(sender); + self.links.push(link_sender); + } + + // creating receiver + let receiver_id = format!( + "receiver-{}-{}-{}-{}", + &self.flow, &self.uuid, &l.to.id, &l.to.input + ); + let receiver = ZFConnectorRecord { + kind: ZFConnectorKind::Receiver, + id: receiver_id.clone(), + resource: z_resource_name.clone(), + link_id: l.to.input.clone(), + runtime: to_runtime, + }; + + //creating link between receiver and component + let link_receiver = ZFLinkDescriptor { + from: ZFFromEndpoint { + id: receiver_id, + output: receiver.link_id.clone(), + }, + to: l.to.clone(), + size: None, + queueing_policy: None, + priority: None, + }; + + // storing info in the data flow record + self.connectors.push(receiver); + self.links.push(link_receiver); + } + } + + Ok(()) + } + + pub fn from_dataflow_descriptor(d: DataFlowDescriptor) -> ZFResult { + let mut dfr = DataFlowRecord { + uuid: Uuid::nil(), // all 0s uuid, placeholder + flow: d.flow.clone(), + operators: Vec::new(), + sinks: Vec::new(), + sources: Vec::new(), + connectors: Vec::new(), + links: Vec::new(), + }; + + for o in &d.operators { + match d.get_mapping(&o.id) { + Some(m) => { + let or = ZFOperatorRecord { + id: o.id.clone(), + // name: o.name.clone(), + inputs: o.inputs.clone(), + outputs: o.outputs.clone(), + uri: o.uri.clone(), + configuration: o.configuration.clone(), + runtime: m, + }; + dfr.operators.push(or) + } + None => { + return Err(ZFError::Uncompleted(format!( + "Missing mapping for {}", + o.id.clone() + ))) + } + } + } + + for s in &d.sources { + match d.get_mapping(&s.id) { + Some(m) => { + let sr = ZFSourceRecord { + id: s.id.clone(), + // name: s.name.clone(), + output: s.output.clone(), + uri: s.uri.clone(), + configuration: s.configuration.clone(), + runtime: m, + }; + dfr.sources.push(sr) + } + None => { + return Err(ZFError::Uncompleted(format!( + "Missing mapping for {}", + s.id.clone() + ))) + } + } + } + + for s in &d.sinks { + match d.get_mapping(&s.id) { + Some(m) => { + let sr = ZFSinkRecord { + id: s.id.clone(), + // name: s.name.clone(), + input: s.input.clone(), + uri: s.uri.clone(), + configuration: s.configuration.clone(), + runtime: m, + }; + dfr.sinks.push(sr) + } + None => { + return Err(ZFError::Uncompleted(format!( + "Missing mapping for {}", + s.id.clone() + ))) + } + } + } + + dfr.add_links(&d.links)?; + Ok(dfr) + } +} diff --git a/zenoh-flow/src/model/link.rs b/zenoh-flow/src/model/link.rs new file mode 100644 index 00000000..ae6fed9a --- /dev/null +++ b/zenoh-flow/src/model/link.rs @@ -0,0 +1,37 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::{types::ZFLinkId, ZFOperatorId}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFLinkDescriptor { + pub from: ZFFromEndpoint, + pub to: ZFToEndpoint, + pub size: Option, + pub queueing_policy: Option, + pub priority: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFFromEndpoint { + pub id: ZFOperatorId, + pub output: ZFLinkId, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFToEndpoint { + pub id: ZFOperatorId, + pub input: ZFLinkId, +} diff --git a/zenoh-flow/src/model/mod.rs b/zenoh-flow/src/model/mod.rs new file mode 100644 index 00000000..acae525e --- /dev/null +++ b/zenoh-flow/src/model/mod.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +pub mod connector; +pub mod dataflow; +pub mod link; +pub mod operator; diff --git a/zenoh-flow/src/model/operator.rs b/zenoh-flow/src/model/operator.rs new file mode 100644 index 00000000..ad9ae500 --- /dev/null +++ b/zenoh-flow/src/model/operator.rs @@ -0,0 +1,113 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::types::{ZFLinkId, ZFOperatorId, ZFRuntimeID}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +// Descriptors + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFSinkDescriptor { + pub id: ZFOperatorId, + pub input: ZFLinkId, + pub uri: Option, + pub configuration: Option>, + pub runtime: Option, // to be removed +} + +impl std::fmt::Display for ZFSinkDescriptor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Sink", self.id) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFSourceDescriptor { + pub id: ZFOperatorId, + pub output: ZFLinkId, + pub uri: Option, + pub configuration: Option>, + pub runtime: Option, // to be removed +} + +impl std::fmt::Display for ZFSourceDescriptor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Source", self.id) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFOperatorDescriptor { + pub id: ZFOperatorId, + pub inputs: Vec, + pub outputs: Vec, + pub uri: Option, + pub configuration: Option>, + pub runtime: Option, // to be removed +} + +impl std::fmt::Display for ZFOperatorDescriptor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Operator", self.id) + } +} + +// Records + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFSinkRecord { + pub id: ZFOperatorId, + pub input: ZFLinkId, + pub uri: Option, + pub configuration: Option>, + pub runtime: ZFRuntimeID, +} + +impl std::fmt::Display for ZFSinkRecord { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Sink", self.id) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFSourceRecord { + pub id: ZFOperatorId, + pub output: ZFLinkId, + pub uri: Option, + pub configuration: Option>, + pub runtime: ZFRuntimeID, +} + +impl std::fmt::Display for ZFSourceRecord { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Source", self.id) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ZFOperatorRecord { + pub id: ZFOperatorId, + pub inputs: Vec, + pub outputs: Vec, + pub uri: Option, + pub configuration: Option>, + pub runtime: ZFRuntimeID, +} + +impl std::fmt::Display for ZFOperatorRecord { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{} - Kind: Operator", self.id) + } +} diff --git a/zenoh-flow/src/runtime/connectors.rs b/zenoh-flow/src/runtime/connectors.rs new file mode 100644 index 00000000..f73a67ee --- /dev/null +++ b/zenoh-flow/src/runtime/connectors.rs @@ -0,0 +1,128 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::async_std::sync::Arc; +use crate::runtime::graph::link::{ZFLinkReceiver, ZFLinkSender}; +use crate::runtime::message::{Message, ZFDataMessage, ZFMessage}; +use crate::{ZFError, ZFResult}; +use futures::prelude::*; +use zenoh::net::{Reliability, Session, SubInfo, SubMode}; + +pub struct ZFZenohSender { + pub session: Arc, + pub resource: String, + pub input: Option>, +} + +impl ZFZenohSender { + pub fn new( + session: Arc, + resource: String, + input: Option>, + ) -> Self { + Self { + session, + resource, + input, + } + } + + pub async fn run(&mut self) -> ZFResult<()> { + log::debug!("ZenohSender - {} - Started", self.resource); + if let Some(mut input) = self.input.take() { + while let Ok((_, msg)) = input.recv().await { + log::debug!("ZenohSender IN <= {:?} ", msg); + + let serialized = match &msg.msg { + Message::Data(data_msg) => match data_msg { + ZFDataMessage::Deserialized(de) => { + let se = Arc::new( + bincode::serialize(&**de) + .map_err(|_| ZFError::SerializationError)?, + ); + let se_msg = ZFMessage { + ts: msg.ts, + msg: Message::Data(ZFDataMessage::new_serialized(se)), + }; + bincode::serialize(&se_msg).map_err(|_| ZFError::SerializationError)? + } + _ => bincode::serialize(&*msg).map_err(|_| ZFError::SerializationError)?, + }, + _ => bincode::serialize(&*msg).map_err(|_| ZFError::SerializationError)?, + }; + log::debug!("ZenohSender - {}=>{:?} ", self.resource, serialized); + self.session + .write(&self.resource.clone().into(), serialized.into()) + .await?; + } + return Err(ZFError::Disconnected); + } + Err(ZFError::Disconnected) + } + + pub fn add_input(&mut self, input: ZFLinkReceiver) { + self.input = Some(input); + } +} + +pub struct ZFZenohReceiver { + pub session: Arc, + pub resource: String, + pub output: Option>, +} + +impl ZFZenohReceiver { + pub fn new( + session: Arc, + resource: String, + output: Option>, + ) -> Self { + Self { + session, + resource, + output, + } + } + + pub async fn run(&mut self) -> ZFResult<()> { + log::debug!("ZenohReceiver - {} - Started", self.resource); + + if let Some(output) = &self.output { + let sub_info = SubInfo { + reliability: Reliability::Reliable, + mode: SubMode::Push, + period: None, + }; + + let mut subscriber = self + .session + .declare_subscriber(&self.resource.clone().into(), &sub_info) + .await?; + + while let Some(msg) = subscriber.receiver().next().await { + log::debug!("ZenohSender - {}<={:?} ", self.resource, msg); + let de: ZFMessage = bincode::deserialize(&msg.payload.contiguous()) + .map_err(|_| ZFError::DeseralizationError)?; + log::debug!("ZenohSender - OUT =>{:?} ", de); + output.send(Arc::new(de)).await?; + } + return Err(ZFError::Disconnected); + } + Err(ZFError::Disconnected) + } + + pub fn add_output(&mut self, output: ZFLinkSender) { + self.output = Some(output); + } +} diff --git a/zenoh-flow/src/runtime/graph/link.rs b/zenoh-flow/src/runtime/graph/link.rs new file mode 100644 index 00000000..4ad0d025 --- /dev/null +++ b/zenoh-flow/src/runtime/graph/link.rs @@ -0,0 +1,157 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use async_std::sync::Arc; + +use crate::types::ZFLinkId; +use crate::ZFResult; + +#[derive(Clone, Debug)] +pub struct ZFLinkSender { + pub id: ZFLinkId, + pub sender: flume::Sender>, +} + +#[derive(Clone, Debug)] +pub struct ZFLinkReceiver { + pub id: ZFLinkId, + pub receiver: flume::Receiver>, + pub last_message: Option>, +} + +impl ZFLinkReceiver { + // pub async fn peek(&mut self) -> ZFResult<(ZFLinkId, Arc)> { + // match &self.last_message { + // Some(message) => Ok((self.id, message.clone())), + // None => { + // let last_message = self.receiver.recv_async().await?; + // self.last_message = Some(last_message.clone()); + + // Ok((self.id, last_message)) + // } + // } + // } + + pub fn peek( + &mut self, + ) -> ::core::pin::Pin)>> + '_>> + { + async fn __peek(_self: &mut ZFLinkReceiver) -> ZFResult<(ZFLinkId, Arc)> { + match &_self.last_message { + Some(message) => Ok((_self.id.clone(), message.clone())), + None => { + let last_message = _self.receiver.recv_async().await?; + _self.last_message = Some(last_message.clone()); + + Ok((_self.id.clone(), last_message)) + } + } + } + Box::pin(__peek(self)) + } + + // pub async fn recv(&mut self) -> ZFResult> { + // match &self.last_message { + // Some(message) => { + // let msg = message.clone(); + // self.last_message = None; + + // Ok(msg) + // } + // None => Ok(self.receiver.recv_async().await?), + // } + // } + + pub fn try_recv(&mut self) -> ZFResult<(ZFLinkId, Arc)> { + match &self.last_message { + Some(message) => { + let msg = message.clone(); + self.last_message = None; + + Ok((self.id.clone(), msg)) + } + None => Ok((self.id.clone(), self.receiver.try_recv()?)), + } + } + + pub fn recv( + &mut self, + ) -> ::core::pin::Pin< + Box)>> + '_ + Send + Sync>, + > { + async fn __recv(_self: &mut ZFLinkReceiver) -> ZFResult<(ZFLinkId, Arc)> { + match &_self.last_message { + Some(message) => { + let msg = message.clone(); + _self.last_message = None; + + Ok((_self.id.clone(), msg)) + } + None => Ok((_self.id.clone(), _self.receiver.recv_async().await?)), + } + } + + Box::pin(__recv(self)) + } + + pub fn drop(&mut self) -> ZFResult<()> { + self.last_message = None; + Ok(()) + } + + pub fn id(&self) -> ZFLinkId { + self.id.clone() + } +} + +impl ZFLinkSender { + pub async fn send(&self, data: Arc) -> ZFResult<()> { + Ok(self.sender.send_async(data).await?) + } + + pub fn len(&self) -> usize { + self.sender.len() + } + + pub fn is_empty(&self) -> bool { + self.sender.is_empty() + } + + pub fn capacity(&self) -> Option { + self.sender.capacity() + } + + pub fn id(&self) -> ZFLinkId { + self.id.clone() + } +} + +pub fn link(capacity: Option, id: ZFLinkId) -> (ZFLinkSender, ZFLinkReceiver) { + let (sender, receiver) = match capacity { + None => flume::unbounded(), + Some(cap) => flume::bounded(cap), + }; + + ( + ZFLinkSender { + id: id.clone(), + sender, + }, + ZFLinkReceiver { + id, + receiver, + last_message: None, + }, + ) +} diff --git a/zenoh-flow/src/runtime/graph/mod.rs b/zenoh-flow/src/runtime/graph/mod.rs new file mode 100644 index 00000000..2339a212 --- /dev/null +++ b/zenoh-flow/src/runtime/graph/mod.rs @@ -0,0 +1,496 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +pub mod link; +pub mod node; + +use async_std::sync::{Arc, Mutex}; +use node::DataFlowNode; +use petgraph::dot::{Config, Dot}; +use petgraph::graph::{EdgeIndex, NodeIndex}; +use petgraph::stable_graph::StableGraph; +use petgraph::Direction; +use std::collections::HashMap; +use zenoh::ZFuture; + +use crate::model::dataflow::DataFlowRecord; +use crate::runtime::connectors::{ZFZenohReceiver, ZFZenohSender}; +use crate::runtime::loader::{load_operator, load_sink, load_source}; +use crate::runtime::message::ZFMessage; +use crate::runtime::runner::{Runner, ZFOperatorRunner, ZFSinkRunner, ZFSourceRunner}; +use crate::{ + model::connector::ZFConnectorKind, + model::link::{ZFFromEndpoint, ZFLinkDescriptor, ZFToEndpoint}, + model::operator::{ZFOperatorRecord, ZFSinkRecord, ZFSourceRecord}, + runtime::graph::link::link, + runtime::graph::node::DataFlowNodeKind, + types::{ZFError, ZFLinkId, ZFOperatorId, ZFOperatorName, ZFResult}, +}; +use uuid::Uuid; + +pub struct DataFlowGraph { + pub uuid: Uuid, + pub flow: String, + pub operators: Vec<(NodeIndex, DataFlowNode)>, + pub links: Vec<(EdgeIndex, ZFLinkDescriptor)>, + pub graph: StableGraph, + pub operators_runners: HashMap>, DataFlowNodeKind)>, +} + +impl DataFlowGraph { + pub fn new() -> Self { + Self { + uuid: Uuid::nil(), + flow: "".to_string(), + operators: Vec::new(), + links: Vec::new(), + graph: StableGraph::::new(), + operators_runners: HashMap::new(), + } + } + + pub fn from_dataflow_record(dr: DataFlowRecord) -> ZFResult { + let mut graph = StableGraph::::new(); + let mut operators = Vec::new(); + let mut links: Vec<(EdgeIndex, ZFLinkDescriptor)> = Vec::new(); + for o in dr.operators { + operators.push(( + graph.add_node(DataFlowNode::Operator(o.clone())), + DataFlowNode::Operator(o), + )); + } + + for o in dr.sources { + operators.push(( + graph.add_node(DataFlowNode::Source(o.clone())), + DataFlowNode::Source(o), + )); + } + + for o in dr.sinks { + operators.push(( + graph.add_node(DataFlowNode::Sink(o.clone())), + DataFlowNode::Sink(o), + )); + } + + for o in dr.connectors { + operators.push(( + graph.add_node(DataFlowNode::Connector(o.clone())), + DataFlowNode::Connector(o), + )); + } + + for l in dr.links { + // First check if the LinkId are the same + if l.from.output != l.to.input { + return Err(ZFError::PortIdNotMatching(( + l.from.output.clone(), + l.to.input, + ))); + } + + let (from_index, from_runtime) = + match operators.iter().find(|&(_, o)| o.get_id() == l.from.id) { + Some((idx, op)) => match op.has_output(l.from.output.clone()) { + true => (idx, op.get_runtime()), + false => return Err(ZFError::PortNotFound((l.from.id, l.from.output))), + }, + None => return Err(ZFError::OperatorNotFound(l.from.id)), + }; + + let (to_index, to_runtime) = + match operators.iter().find(|&(_, o)| o.get_id() == l.to.id) { + Some((idx, op)) => match op.has_input(l.to.input.clone()) { + true => (idx, op.get_runtime()), + false => return Err(ZFError::PortNotFound((l.to.id, l.to.input))), + }, + None => return Err(ZFError::OperatorNotFound(l.to.id)), + }; + + if from_runtime == to_runtime { + log::debug!("[Graph instantiation] [same runtime] Pushing link: {:?}", l); + links.push(( + graph.add_edge(*from_index, *to_index, l.from.output.clone()), + l.clone(), + )); + } else { + log::debug!( + "[Graph instantiation] Link on different runtime detected: {:?}, this should not happen! :P", + l + ); + + // We do nothing in this case... the links are already well created when creating the record, so this should NEVER happen + } + } + + Ok(Self { + uuid: dr.uuid, + flow: dr.flow, + operators, + links, + graph, + operators_runners: HashMap::new(), + }) + } + + pub fn set_name(&mut self, name: String) { + self.flow = name; + } + + pub fn to_dot_notation(&self) -> String { + format!("{}", Dot::with_config(&self.graph, &[Config::EdgeNoLabel])) + } + + pub fn add_static_operator( + &mut self, + id: ZFOperatorId, + inputs: Vec, + outputs: Vec, + operator: Box, + configuration: Option>, + ) -> ZFResult<()> { + let descriptor = ZFOperatorRecord { + id: id.clone(), + inputs, + outputs, + uri: None, + configuration, + runtime: String::from("self"), + }; + self.operators.push(( + self.graph + .add_node(DataFlowNode::Operator(descriptor.clone())), + DataFlowNode::Operator(descriptor), + )); + let runner = Runner::Operator(ZFOperatorRunner::new(operator, None)); + self.operators_runners.insert( + id, + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Operator), + ); + Ok(()) + } + + pub fn add_static_source( + &mut self, + id: ZFOperatorId, + output: ZFLinkId, + source: Box, + configuration: Option>, + ) -> ZFResult<()> { + let descriptor = ZFSourceRecord { + id: id.clone(), + output, + uri: None, + configuration, + runtime: String::from("self"), + }; + self.operators.push(( + self.graph + .add_node(DataFlowNode::Source(descriptor.clone())), + DataFlowNode::Source(descriptor), + )); + let runner = Runner::Source(ZFSourceRunner::new(source, None)); + self.operators_runners + .insert(id, (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Source)); + Ok(()) + } + + pub fn add_static_sink( + &mut self, + id: ZFOperatorId, + input: ZFLinkId, + sink: Box, + configuration: Option>, + ) -> ZFResult<()> { + let descriptor = ZFSinkRecord { + id: id.clone(), + input, + uri: None, + configuration, + runtime: String::from("self"), + }; + self.operators.push(( + self.graph.add_node(DataFlowNode::Sink(descriptor.clone())), + DataFlowNode::Sink(descriptor), + )); + let runner = Runner::Sink(ZFSinkRunner::new(sink, None)); + self.operators_runners + .insert(id, (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Sink)); + Ok(()) + } + + pub fn add_link( + &mut self, + from: ZFFromEndpoint, + to: ZFToEndpoint, + size: Option, + queueing_policy: Option, + priority: Option, + ) -> ZFResult<()> { + let connection = ZFLinkDescriptor { + from, + to, + size, + queueing_policy, + priority, + }; + + if connection.from.output == connection.to.input { + let from_index = match self + .operators + .iter() + .find(|&(_, o)| o.get_id() == connection.from.id.clone()) + { + Some((idx, op)) => match op.has_output(connection.from.output.clone()) { + true => idx, + false => { + return Err(ZFError::PortNotFound(( + connection.from.id.clone(), + connection.from.output.clone(), + ))) + } + }, + None => return Err(ZFError::OperatorNotFound(connection.from.id.clone())), + }; + + let to_index = match self + .operators + .iter() + .find(|&(_, o)| o.get_id() == connection.to.id.clone()) + { + Some((idx, op)) => match op.has_input(connection.to.input.clone()) { + true => idx, + false => { + return Err(ZFError::PortNotFound(( + connection.to.id.clone(), + connection.to.input.clone(), + ))) + } + }, + None => return Err(ZFError::OperatorNotFound(connection.to.id.clone())), + }; + + self.links.push(( + self.graph + .add_edge(*from_index, *to_index, connection.from.output.clone()), + connection, + )); + } else { + return Err(ZFError::PortIdNotMatching(( + connection.from.output.clone(), + connection.to.input, + ))); + } + + Ok(()) + } + + pub fn load(&mut self, runtime: &str) -> ZFResult<()> { + let session = Arc::new(zenoh::net::open(zenoh::net::config::peer()).wait()?); + for (_, op) in &self.operators { + if op.get_runtime() != runtime { + continue; + } + + match op { + DataFlowNode::Operator(inner) => { + match &inner.uri { + Some(uri) => { + let runner = load_operator(uri.clone(), inner.configuration.clone())?; + let runner = Runner::Operator(runner); + self.operators_runners.insert( + inner.id.clone(), + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Operator), + ); + } + None => { + // this is a static operator. + } + } + } + DataFlowNode::Source(inner) => { + match &inner.uri { + Some(uri) => { + let runner = load_source(uri.clone(), inner.configuration.clone())?; + let runner = Runner::Source(runner); + self.operators_runners.insert( + inner.id.clone(), + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Source), + ); + } + None => { + // static source + } + } + } + DataFlowNode::Sink(inner) => { + match &inner.uri { + Some(uri) => { + let runner = load_sink(uri.clone(), inner.configuration.clone())?; + let runner = Runner::Sink(runner); + self.operators_runners.insert( + inner.id.clone(), + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Sink), + ); + } + None => { + //static sink + } + } + } + DataFlowNode::Connector(zc) => match zc.kind { + ZFConnectorKind::Sender => { + let runner = ZFZenohSender::new(session.clone(), zc.resource.clone(), None); + let runner = Runner::Sender(runner); + self.operators_runners.insert( + zc.id.clone(), + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Connector), + ); + } + + ZFConnectorKind::Receiver => { + let runner = + ZFZenohReceiver::new(session.clone(), zc.resource.clone(), None); + let runner = Runner::Receiver(runner); + self.operators_runners.insert( + zc.id.clone(), + (Arc::new(Mutex::new(runner)), DataFlowNodeKind::Connector), + ); + } + }, + } + } + Ok(()) + } + + pub async fn make_connections(&mut self, runtime: &str) -> ZFResult<()> { + // Connects the operators via our FIFOs + + for (idx, up_op) in &self.operators { + if up_op.get_runtime() != runtime { + continue; + } + + log::debug!("Creating links for:\n\t< {:?} > Operator: {:?}", idx, up_op); + + let (up_runner, _) = self + .operators_runners + .get(&up_op.get_id()) + .ok_or(ZFError::OperatorNotFound(up_op.get_id().clone()))?; + let mut up_runner = up_runner.lock().await; + + if self.graph.contains_node(*idx) { + let mut downstreams = self + .graph + .neighbors_directed(*idx, Direction::Outgoing) + .detach(); + while let Some((down_edge_index, down_node_index)) = downstreams.next(&self.graph) { + let (_, down_link) = self + .links + .iter() + .find(|&(edge_index, _)| *edge_index == down_edge_index) + .ok_or(ZFError::OperatorNotFound(up_op.get_id().clone()))?; + let link_id = down_link.from.output.clone(); + + let down_op = match self + .operators + .iter() + .find(|&(idx, _)| *idx == down_node_index) + { + Some((_, op)) => op, + None => panic!("To not found"), + }; + + let (down_runner, _) = self + .operators_runners + .get(&down_op.get_id()) + .ok_or(ZFError::OperatorNotFound(down_op.get_id().clone()))?; + let mut down_runner = down_runner.lock().await; + + log::debug!( + "\t Creating link between {:?} -> {:?}: {:?}", + idx, + down_node_index, + link_id + ); + let (tx, rx) = link::(None, link_id); + + up_runner.add_output(tx); + down_runner.add_input(rx); + } + } + } + Ok(()) + } + + pub fn get_runner(&self, operator_id: &ZFOperatorId) -> Option>> { + self.operators_runners + .get(operator_id) + .map(|(r, _)| r.clone()) + } + + pub fn get_runners(&self) -> Vec>> { + let mut runners = vec![]; + + for (runner, _) in self.operators_runners.values() { + runners.push(runner.clone()); + } + runners + } + + pub fn get_sources(&self) -> Vec>> { + let mut runners = vec![]; + + for (runner, kind) in self.operators_runners.values() { + if let DataFlowNodeKind::Source = kind { + runners.push(runner.clone()); + } + } + runners + } + + pub fn get_sinks(&self) -> Vec>> { + let mut runners = vec![]; + + for (runner, kind) in self.operators_runners.values() { + if let DataFlowNodeKind::Sink = kind { + runners.push(runner.clone()); + } + } + runners + } + + pub fn get_operators(&self) -> Vec>> { + let mut runners = vec![]; + + for (runner, kind) in self.operators_runners.values() { + if let DataFlowNodeKind::Operator = kind { + runners.push(runner.clone()); + } + } + runners + } + + pub fn get_connectors(&self) -> Vec>> { + let mut runners = vec![]; + + for (runner, kind) in self.operators_runners.values() { + if let DataFlowNodeKind::Connector = kind { + runners.push(runner.clone()); + } + } + runners + } +} diff --git a/zenoh-flow/src/runtime/graph/node.rs b/zenoh-flow/src/runtime/graph/node.rs new file mode 100644 index 00000000..9599950d --- /dev/null +++ b/zenoh-flow/src/runtime/graph/node.rs @@ -0,0 +1,101 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::model::connector::{ZFConnectorKind, ZFConnectorRecord}; +use crate::model::operator::{ZFOperatorRecord, ZFSinkRecord, ZFSourceRecord}; +use crate::{ZFLinkId, ZFOperatorId, ZFRuntimeID}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum DataFlowNodeKind { + Operator, + Source, + Sink, + Connector, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum DataFlowNode { + Operator(ZFOperatorRecord), + Source(ZFSourceRecord), + Sink(ZFSinkRecord), + Connector(ZFConnectorRecord), +} + +impl std::fmt::Display for DataFlowNode { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + DataFlowNode::Operator(inner) => write!(f, "{}", inner), + DataFlowNode::Source(inner) => write!(f, "{}", inner), + DataFlowNode::Sink(inner) => write!(f, "{}", inner), + DataFlowNode::Connector(inner) => write!(f, "{}", inner), + } + } +} + +impl DataFlowNode { + pub fn has_input(&self, id: ZFLinkId) -> bool { + match self { + DataFlowNode::Operator(op) => match op.inputs.iter().find(|&lid| *lid == id) { + Some(_lid) => true, + None => false, + }, + DataFlowNode::Source(_) => false, + DataFlowNode::Sink(sink) => sink.input == id, + DataFlowNode::Connector(zc) => match zc.kind { + ZFConnectorKind::Receiver => false, + ZFConnectorKind::Sender => zc.link_id == id, + }, + } + } + + pub fn has_output(&self, id: ZFLinkId) -> bool { + match self { + DataFlowNode::Operator(op) => match op.outputs.iter().find(|&lid| *lid == id) { + Some(_lid) => true, + None => false, + }, + DataFlowNode::Sink(_) => false, + DataFlowNode::Source(source) => source.output == id, + DataFlowNode::Connector(zc) => match zc.kind { + ZFConnectorKind::Receiver => zc.link_id == id, + ZFConnectorKind::Sender => false, + }, + } + } + + pub fn get_id(&self) -> ZFOperatorId { + match self { + DataFlowNode::Operator(op) => op.id.clone(), + DataFlowNode::Sink(s) => s.id.clone(), + DataFlowNode::Source(s) => s.id.clone(), + DataFlowNode::Connector(zc) => match zc.kind { + ZFConnectorKind::Receiver => zc.id.clone(), + ZFConnectorKind::Sender => zc.id.clone(), + }, + } + } + + pub fn get_runtime(&self) -> ZFRuntimeID { + match self { + DataFlowNode::Operator(op) => op.runtime.clone(), + DataFlowNode::Sink(s) => s.runtime.clone(), + DataFlowNode::Source(s) => s.runtime.clone(), + DataFlowNode::Connector(zc) => match zc.kind { + ZFConnectorKind::Receiver => zc.runtime.clone(), + ZFConnectorKind::Sender => zc.runtime.clone(), + }, + } + } +} diff --git a/zenoh-flow/src/runtime/loader.rs b/zenoh-flow/src/runtime/loader.rs new file mode 100644 index 00000000..beae7d97 --- /dev/null +++ b/zenoh-flow/src/runtime/loader.rs @@ -0,0 +1,142 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::runtime::runner::{ + ZFOperatorDeclaration, ZFOperatorRunner, ZFSinkDeclaration, ZFSinkRunner, ZFSourceDeclaration, + ZFSourceRunner, +}; +use crate::types::{ZFError, ZFResult}; +use libloading::Library; +use std::collections::HashMap; +use url::Url; + +pub static CORE_VERSION: &str = env!("CARGO_PKG_VERSION"); +pub static RUSTC_VERSION: &str = env!("RUSTC_VERSION"); + +// OPERATOR + +/// # Safety +/// +/// TODO remove all copy-pasted code, make macros/functions instead +pub fn load_operator( + path: String, + configuration: Option>, +) -> ZFResult { + let uri = Url::parse(&path).map_err(|err| ZFError::ParsingError(format!("{}", err)))?; + + match uri.scheme() { + "file" => unsafe { load_lib_operator(make_file_path(uri), configuration) }, + _ => Err(ZFError::Unimplemented), + } +} + +pub unsafe fn load_lib_operator( + path: String, + configuration: Option>, +) -> ZFResult { + log::debug!("Operator Loading {}", path); + + let library = Library::new(path)?; + let decl = library + .get::<*mut ZFOperatorDeclaration>(b"zfoperator_declaration\0")? + .read(); + + // version checks to prevent accidental ABI incompatibilities + if decl.rustc_version != RUSTC_VERSION || decl.core_version != CORE_VERSION { + return Err(ZFError::VersionMismatch); + } + + let operator = (decl.register)(configuration)?; + + let runner = ZFOperatorRunner::new(operator, Some(library)); + Ok(runner) +} + +// SOURCE + +pub fn load_source( + path: String, + configuration: Option>, +) -> ZFResult { + let uri = Url::parse(&path).map_err(|err| ZFError::ParsingError(format!("{}", err)))?; + + match uri.scheme() { + "file" => unsafe { load_lib_source(make_file_path(uri), configuration) }, + _ => Err(ZFError::Unimplemented), + } +} + +pub unsafe fn load_lib_source( + path: String, + configuration: Option>, +) -> ZFResult { + log::debug!("Source Loading {}", path); + let library = Library::new(path)?; + let decl = library + .get::<*mut ZFSourceDeclaration>(b"zfsource_declaration\0")? + .read(); + + // version checks to prevent accidental ABI incompatibilities + if decl.rustc_version != RUSTC_VERSION || decl.core_version != CORE_VERSION { + return Err(ZFError::VersionMismatch); + } + + let source = (decl.register)(configuration)?; + + let runner = ZFSourceRunner::new(source, Some(library)); + Ok(runner) +} + +// SINK + +pub fn load_sink( + path: String, + configuration: Option>, +) -> ZFResult { + let uri = Url::parse(&path).map_err(|err| ZFError::ParsingError(format!("{}", err)))?; + + match uri.scheme() { + "file" => unsafe { load_lib_sink(make_file_path(uri), configuration) }, + _ => Err(ZFError::Unimplemented), + } +} + +pub unsafe fn load_lib_sink( + path: String, + configuration: Option>, +) -> ZFResult { + log::debug!("Sink Loading {}", path); + let library = Library::new(path)?; + + let decl = library + .get::<*mut ZFSinkDeclaration>(b"zfsink_declaration\0")? + .read(); + + // version checks to prevent accidental ABI incompatibilities + if decl.rustc_version != RUSTC_VERSION || decl.core_version != CORE_VERSION { + return Err(ZFError::VersionMismatch); + } + + let sink = (decl.register)(configuration)?; + + let runner = ZFSinkRunner::new(sink, Some(library)); + Ok(runner) +} + +pub fn make_file_path(uri: Url) -> String { + match uri.host_str() { + Some(h) => format!("{}{}", h, uri.path()), + None => format!("{}", uri.path()), + } +} diff --git a/zenoh-flow/src/runtime/message.rs b/zenoh-flow/src/runtime/message.rs new file mode 100644 index 00000000..4cd34742 --- /dev/null +++ b/zenoh-flow/src/runtime/message.rs @@ -0,0 +1,101 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +extern crate serde; + +use crate::{DataTrait, ZFTimestamp}; +use async_std::sync::Arc; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +//TODO: improve +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum ZFDataMessage { + Serialized(Arc>), + #[serde(skip_serializing, skip_deserializing)] + // Deserialized data is never serialized directly + Deserialized(Arc), +} + +impl ZFDataMessage { + pub fn new_serialized(data: Arc>) -> Self { + Self::Serialized(data) + } + + pub fn serialized_data(&self) -> &[u8] { + match self { + Self::Serialized(data) => &data, + _ => panic!(), + } + } + + pub fn new_deserialized(data: Arc) -> Self { + Self::Deserialized(data) + } + + pub fn deserialized_data(&self) -> Arc { + match self { + Self::Deserialized(data) => data.clone(), + _ => panic!(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum ZFCtrlMessage { + ReadyToMigrate, + ChangeMode(u8, u128), + Watermark, +} + +//TODO: improve, change name +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Message { + Data(ZFDataMessage), + Ctrl(ZFCtrlMessage), +} + +//TODO: improve +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ZFMessage { + pub ts: ZFTimestamp, + pub msg: Message, +} + +impl ZFMessage { + pub fn from_raw(data: Arc>) -> Self { + Self { + ts: 0, //placeholder + msg: Message::Data(ZFDataMessage::new_serialized(data)), + } + } + + pub fn from_data(data: Arc) -> Self { + Self { + ts: 0, //placeholder + msg: Message::Data(ZFDataMessage::new_deserialized(data)), + } + } + + pub fn from_message(msg: ZFDataMessage) -> Self { + Self { + ts: 0, //placeholder + msg: Message::Data(msg), + } + } + + pub fn timestamp(&self) -> &ZFTimestamp { + &self.ts + } +} diff --git a/zenoh-flow/src/runtime/mod.rs b/zenoh-flow/src/runtime/mod.rs new file mode 100644 index 00000000..4d18c564 --- /dev/null +++ b/zenoh-flow/src/runtime/mod.rs @@ -0,0 +1,81 @@ +use crate::{ + model::dataflow::{DataFlowDescriptor, Mapping}, + ZFResult, ZFRuntimeID, +}; + +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +pub mod connectors; +pub mod graph; +pub mod loader; +pub mod message; +pub mod runner; + +pub async fn map_to_infrastructure( + mut descriptor: DataFlowDescriptor, + runtime: &ZFRuntimeID, +) -> ZFResult { + log::debug!("[Dataflow mapping] Begin mapping for: {}", descriptor.flow); + + // Initial "stupid" mapping, if an operator is not mapped, we map to the local runtime. + // function is async because it could involve other nodes. + + let mut mappings = Vec::new(); + + for o in &descriptor.operators { + match descriptor.get_mapping(&o.id) { + Some(_) => (), + None => { + let mapping = Mapping { + id: o.id.clone(), + runtime: (*runtime).clone(), + }; + mappings.push(mapping); + } + } + } + + for o in &descriptor.sources { + match descriptor.get_mapping(&o.id) { + Some(_) => (), + None => { + let mapping = Mapping { + id: o.id.clone(), + runtime: (*runtime).clone(), + }; + mappings.push(mapping); + } + } + } + + for o in &descriptor.sinks { + match descriptor.get_mapping(&o.id) { + Some(_) => (), + None => { + let mapping = Mapping { + id: o.id.clone(), + runtime: (*runtime).clone(), + }; + mappings.push(mapping); + } + } + } + + for m in mappings { + descriptor.add_mapping(m) + } + + Ok(descriptor) +} diff --git a/zenoh-flow/src/runtime/runner.rs b/zenoh-flow/src/runtime/runner.rs new file mode 100644 index 00000000..344b3852 --- /dev/null +++ b/zenoh-flow/src/runtime/runner.rs @@ -0,0 +1,373 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::async_std::sync::Arc; +use crate::runtime::connectors::{ZFZenohReceiver, ZFZenohSender}; +use crate::runtime::graph::link::{ZFLinkReceiver, ZFLinkSender}; +use crate::runtime::message::{Message, ZFMessage}; +use crate::types::{Token, ZFContext, ZFData, ZFInput, ZFLinkId, ZFResult}; +use crate::{OperatorTrait, SinkTrait, SourceTrait}; +use futures::future; +use libloading::Library; +use std::collections::HashMap; +use zenoh::net::Session; + +pub enum Runner { + Operator(ZFOperatorRunner), + Source(ZFSourceRunner), + Sink(ZFSinkRunner), + Sender(ZFZenohSender), + Receiver(ZFZenohReceiver), +} + +impl Runner { + pub async fn run(&mut self) -> ZFResult<()> { + match self { + Runner::Operator(runner) => runner.run().await, + Runner::Source(runner) => runner.run().await, + Runner::Sink(runner) => runner.run().await, + Runner::Sender(runner) => runner.run().await, + Runner::Receiver(runner) => runner.run().await, + } + } + + pub fn add_input(&mut self, input: ZFLinkReceiver) { + match self { + Runner::Operator(runner) => runner.add_input(input), + Runner::Source(_) => panic!("Sources does not have inputs!"), // TODO this should return a ZFResult<()> + Runner::Sink(runner) => runner.add_input(input), + Runner::Sender(runner) => runner.add_input(input), + Runner::Receiver(_) => panic!("Receiver does not have inputs!"), // TODO this should return a ZFResult<()> + } + } + + pub fn add_output(&mut self, output: ZFLinkSender) { + match self { + Runner::Operator(runner) => runner.add_output(output), + Runner::Source(runner) => runner.add_output(output), + Runner::Sink(_) => panic!("Sinks does not have output!"), // TODO this should return a ZFResult<()> + Runner::Sender(_) => panic!("Senders does not have output!"), // TODO this should return a ZFResult<()> + Runner::Receiver(runner) => runner.add_output(output), + } + } +} + +#[repr(C)] +pub struct ZFOperatorDeclaration { + pub rustc_version: &'static str, + pub core_version: &'static str, + pub register: unsafe extern "C" fn( + Option>, + ) -> ZFResult>, +} + +pub struct ZFOperatorRunner { + pub operator: Box, + pub lib: Option, + pub inputs: Vec>, + pub outputs: HashMap>>, +} + +impl ZFOperatorRunner { + pub fn new(operator: Box, lib: Option) -> Self { + Self { + operator, + lib, + inputs: vec![], + outputs: HashMap::new(), + } + } + + pub fn add_input(&mut self, input: ZFLinkReceiver) { + self.inputs.push(input); + } + + pub fn add_output(&mut self, output: ZFLinkSender) { + let key = output.id(); + if let Some(links) = self.outputs.get_mut(&key) { + links.push(output); + } else { + self.outputs.insert(key, vec![output]); + } + } + + pub async fn run(&mut self) -> ZFResult<()> { + // WIP empty context + let ctx = ZFContext::new(self.operator.get_state(), 0); + + loop { + // we should start from an HashMap with all ZFLinkId and not ready tokens + let mut msgs: HashMap = HashMap::new(); + + for i in &self.inputs { + msgs.insert(i.id(), Token::new_not_ready(0)); + } + + let ir_fn = self.operator.get_input_rule(ctx.clone()); + + let mut futs = vec![]; + for rx in self.inputs.iter_mut() { + futs.push(rx.recv()); // this should be peek(), but both requires mut + } + + crate::run_input_rules!(ir_fn, msgs, futs, ctx); + + // Running + let run_fn = self.operator.get_run(ctx.clone()); + let mut data = ZFInput::new(); + + for (id, v) in msgs { + let (d, _) = v.split(); + data.insert(id, ZFData::from(d.unwrap())); + } + + let outputs = run_fn(ctx.clone(), data)?; + + // Output + let out_fn = self.operator.get_output_rule(ctx.clone()); + + let out_msgs = out_fn(ctx.clone(), outputs)?; + + // Send to Links + for (id, zf_msg) in out_msgs { + // getting link + log::debug!("id: {:?}, zf_msg: {:?}", id, zf_msg); + if let Some(links) = self.outputs.get(&id) { + for tx in links { + log::debug!("Sending on: {:?}", tx); + match zf_msg.msg { + Message::Data(_) => { + tx.send(zf_msg.clone()).await?; + } + Message::Ctrl(_) => { + // here we process should process control messages (eg. change mode) + // tx.send(zf_msg); + } + } + } + } + } + + // This depends on the Tokens... + for rx in self.inputs.iter_mut() { + rx.drop()?; + } + } + } +} + +pub struct ZFSourceDeclaration { + pub rustc_version: &'static str, + pub core_version: &'static str, + pub register: unsafe extern "C" fn( + Option>, + ) -> ZFResult>, +} + +pub struct ZFZenohReceiverDeclaration { + pub rustc_version: &'static str, + pub core_version: &'static str, + pub register: unsafe extern "C" fn( + Arc, + Option>, + ) -> ZFResult>, +} + +// TODO to be removed +pub trait ZFSourceRegistrarTrait { + fn register_zfsource(&mut self, name: &str, operator: Box); +} + +pub struct ZFSourceRunner { + pub operator: Box, + pub lib: Option, + pub outputs: Vec>, +} + +impl ZFSourceRunner { + pub fn new(operator: Box, lib: Option) -> Self { + Self { + operator, + lib, + outputs: vec![], + } + } + + pub fn add_output(&mut self, output: ZFLinkSender) { + self.outputs.push(output); + } + + pub async fn run(&mut self) -> ZFResult<()> { + // WIP empty context + let ctx = ZFContext::new(self.operator.get_state(), 0); + + loop { + // Running + let run_fn = self.operator.get_run(ctx.clone()); + let outputs = run_fn(ctx.clone()).await?; + + //Output + let out_fn = self.operator.get_output_rule(ctx.clone()); + + let out_msgs = out_fn(ctx.clone(), outputs)?; + log::debug!("Outputs: {:?}", self.outputs); + + // Send to Links + for (id, zf_msg) in out_msgs { + log::debug!("Sending on {:?} data: {:?}", id, zf_msg); + //getting link + let tx = self.outputs.iter().find(|&x| x.id() == id).unwrap(); + //println!("Tx: {:?} Receivers: {:?}", tx.inner.tx, tx.inner.tx.receiver_count()); + match zf_msg.msg { + Message::Data(_) => { + tx.send(zf_msg).await?; + } + Message::Ctrl(_) => { + // here we process should process control messages (eg. change mode) + //tx.send(zf_msg); + } + } + } + } + } +} + +pub struct ZFZenohSenderDeclaration { + pub rustc_version: &'static str, + pub core_version: &'static str, + pub register: unsafe extern "C" fn( + Arc, + Option>, + ) -> ZFResult>, +} + +pub struct ZFSinkDeclaration { + pub rustc_version: &'static str, + pub core_version: &'static str, + pub register: unsafe extern "C" fn( + Option>, + ) -> ZFResult>, +} + +pub trait ZFSinkRegistrarTrait { + fn register_zfsink(&mut self, name: &str, operator: Box); +} + +pub struct ZFSinkRunner { + pub operator: Box, + pub lib: Option, + pub inputs: Vec>, +} + +impl ZFSinkRunner { + pub fn new(operator: Box, lib: Option) -> Self { + Self { + operator, + lib, + inputs: vec![], + } + } + + pub fn add_input(&mut self, input: ZFLinkReceiver) { + self.inputs.push(input); + } + + pub async fn run(&mut self) -> ZFResult<()> { + // WIP empty context + let ctx = ZFContext::new(self.operator.get_state(), 0); + + loop { + // we should start from an HashMap with all ZFLinkId and not ready tokens + let mut msgs: HashMap = HashMap::new(); + + for i in self.inputs.iter() { + msgs.insert(i.id(), Token::new_not_ready(0)); + } + + let ir_fn = self.operator.get_input_rule(ctx.clone()); + + let mut futs = vec![]; + for rx in self.inputs.iter_mut() { + futs.push(rx.recv()); // this should be peek(), but both requires mut + } + + crate::run_input_rules!(ir_fn, msgs, futs, ctx); + + // Running + let run_fn = self.operator.get_run(ctx.clone()); + let mut data = ZFInput::new(); + + for (id, v) in msgs { + log::debug!("[SINK] Sending data to run: {:?}", v); + let (d, _) = v.split(); + if d.is_none() { + continue; + } + data.insert(id, ZFData::from(d.unwrap())); + } + + run_fn(ctx.clone(), data).await?; + + //This depends on the Tokens... + for rx in self.inputs.iter_mut() { + rx.drop()?; + } + } + } +} + +#[macro_export] +macro_rules! run_input_rules { + ($ir : expr, $tokens : expr, $links : expr, $ctx: expr) => { + while !$links.is_empty() { + match future::select_all($links).await { + // this could be "slow" as suggested by LC + (Ok((id, msg)), _i, remaining) => { + match &msg.msg { + Message::Data(_) => { + $tokens.insert(id, Token::from(msg)); + + match $ir($ctx.clone(), &mut $tokens) { + Ok(true) => { + // we can run + log::debug!("IR: OK"); + $links = vec![]; // this makes the while loop to end + } + Ok(false) => { + //we cannot run, we should update the list of futures + log::debug!("IR: Not OK"); + $links = remaining; + } + Err(_) => { + // we got an error on the input rules, we should recover/update list of futures + log::debug!("IR: received an error"); + $links = remaining; + } + } + } + Message::Ctrl(_) => { + //control message receiver, we should handle it + $links = remaining; + } + }; + } + (Err(e), i, remaining) => { + log::debug!("Link index {:?} has got error {:?}", i, e); + $links = remaining; + } + } + }; + drop($links); + }; +} diff --git a/zenoh-flow/src/types.rs b/zenoh-flow/src/types.rs new file mode 100644 index 00000000..8735e546 --- /dev/null +++ b/zenoh-flow/src/types.rs @@ -0,0 +1,466 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use crate::async_std::sync::{Arc, Mutex, MutexGuard}; +use crate::runtime::message::{Message, ZFDataMessage, ZFMessage}; +use crate::serde::{Deserialize, Serialize}; +use futures::Future; +use std::any::Any; +use std::collections::HashMap; +use std::convert::From; +use std::fmt::Debug; +use std::pin::Pin; + +// Placeholder types +pub type ZFOperatorId = String; +pub type ZFZenohResource = String; +pub type ZFOperatorName = String; +pub type ZFTimestamp = usize; //TODO: improve it, usize is just a placeholder +pub type ZFLinkId = String; // TODO: improve it, String is just a placeholder +pub type ZFRuntimeID = String; + +#[derive(Debug, PartialEq)] +pub enum ZFError { + GenericError, + SerializationError, + DeseralizationError, + MissingState, + InvalidState, + Unimplemented, + Empty, + MissingConfiguration, + VersionMismatch, + Disconnected, + Uncompleted(String), + PortIdNotMatching((ZFLinkId, ZFLinkId)), + OperatorNotFound(ZFOperatorId), + PortNotFound((ZFOperatorId, ZFLinkId)), + RecvError(flume::RecvError), + SendError(String), + MissingInput(ZFLinkId), + InvalidData(ZFLinkId), + IOError(String), + ZenohError(String), + LoadingError(String), + ParsingError(String), +} + +impl From for ZFError { + fn from(err: flume::RecvError) -> Self { + Self::RecvError(err) + } +} + +impl From for ZFError { + fn from(err: flume::TryRecvError) -> Self { + match err { + flume::TryRecvError::Disconnected => Self::Disconnected, + flume::TryRecvError::Empty => Self::Empty, + } + } +} + +impl From> for ZFError { + fn from(err: flume::SendError) -> Self { + Self::SendError(format!("{:?}", err)) + } +} + +impl From for ZFError { + fn from(err: std::io::Error) -> Self { + Self::IOError(format!("{}", err)) + } +} + +impl From for ZFError { + fn from(err: zenoh_util::core::ZError) -> Self { + Self::ZenohError(format!("{}", err)) + } +} + +impl From for ZFError { + fn from(err: libloading::Error) -> Self { + Self::LoadingError(format!("Error when loading the library: {}", err)) + } +} + +pub struct ZFInnerCtx { + pub state: Box, + pub mode: usize, //can be arc and inside ZFContext +} + +#[derive(Clone)] +pub struct ZFContext(Arc>); //TODO: have only state inside Mutex + +impl ZFContext { + pub fn new(state: Box, mode: usize) -> Self { + let inner = Arc::new(Mutex::new(ZFInnerCtx { + mode: mode, + state: state, + })); + Self(inner) + } + + pub async fn async_lock<'a>(&'a self) -> MutexGuard<'a, ZFInnerCtx> { + self.0.lock().await + } + + pub fn lock<'a>(&'a self) -> MutexGuard<'a, ZFInnerCtx> { + crate::zf_spin_lock!(self.0) + } +} + +pub type ZFResult = Result; + +#[typetag::serde(tag = "zf_data_type", content = "value")] +pub trait DataTrait: Debug + Send + Sync { + fn as_any(&self) -> &dyn Any; + fn as_mut_any(&mut self) -> &mut dyn Any; +} + +//Create a Derive macro for this +#[typetag::serde(tag = "zf_state_type", content = "value")] +pub trait StateTrait: Debug + Send + Sync { + fn as_any(&self) -> &dyn Any; + fn as_mut_any(&mut self) -> &mut dyn Any; +} + +pub trait OperatorMode: Into + From {} //Placeholder + +pub type ZFSourceResult = Result, ZFError>; + +pub type ZFSourceRun = dyn Fn(&mut ZFContext) -> ZFSourceResult + Send + Sync + 'static; // This should be a future, Sources can do I/O + +pub type ZFSinkResult = Result<(), ZFError>; + +pub type ZFSinkRun = + dyn Fn(&mut ZFContext, Vec<&ZFMessage>) -> ZFSinkResult + Send + Sync + 'static; // This should be a future, Sinks can do I/O + +pub type InputRuleResult = ZFResult; + +// CAUTION, USER CAN DO NASTY THINGS, eg. remove a link we have passed to him. +pub type FnInputRule = + dyn Fn(ZFContext, &mut HashMap) -> InputRuleResult + Send + Sync + 'static; + +pub type OutputRuleResult = ZFResult>>; + +pub type FnOutputRule = dyn Fn(ZFContext, HashMap>) -> OutputRuleResult + + Send + + Sync + + 'static; + +pub type RunResult = ZFResult>>; + +pub type FnRun = dyn Fn(ZFContext, ZFInput) -> RunResult + Send + Sync + 'static; + +pub trait OperatorTrait { + fn get_input_rule(&self, ctx: ZFContext) -> Box; + + fn get_output_rule(&self, ctx: ZFContext) -> Box; + + fn get_run(&self, ctx: ZFContext) -> Box; + + fn get_state(&self) -> Box; +} + +pub type FutRunResult = Pin + Send + Sync>>; + +pub type FnSourceRun = Box FutRunResult + Send + Sync>; + +pub trait SourceTrait { + fn get_run(&self, ctx: ZFContext) -> FnSourceRun; + + fn get_output_rule(&self, ctx: ZFContext) -> Box; + + fn get_state(&self) -> Box; +} + +pub type FutSinkResult = Pin> + Send + Sync>>; + +pub type FnSinkRun = Box FutSinkResult + Send + Sync>; + +pub trait SinkTrait { + fn get_input_rule(&self, ctx: ZFContext) -> Box; + + fn get_run(&self, ctx: ZFContext) -> FnSinkRun; + + fn get_state(&self) -> Box; +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TokenAction { + Consume, // Default, data is passed to the run and consumed from the "thing" + Drop, // Data is dropped + KeepRun, // Data is passed to the run and kept in the "thing" + Keep, // Data is kept in the "thing" + Wait, //Waits the Data, this is applicable only to NotReadyToken +} + +#[derive(Debug, Clone, Default)] +pub struct NotReadyToken { + pub ts: ZFTimestamp, +} + +impl NotReadyToken { + /// Creates a `NotReadyToken` with its timestamp set to 0. + pub fn new() -> Self { + NotReadyToken { ts: 0 } + } +} + +#[derive(Debug, Clone)] +pub enum TokenData { + Serialized(Arc>), + Deserialized(Arc), +} + +impl TokenData { + pub fn get_data(self) -> ZFResult> { + match self { + Self::Serialized(_) => Err(ZFError::GenericError), + Self::Deserialized(de) => Ok(de), + } + } + + pub fn get_raw(self) -> ZFResult>> { + match self { + Self::Serialized(ser) => Ok(ser), + Self::Deserialized(_) => Err(ZFError::GenericError), + } + } +} + +//TODO: improve +#[derive(Debug, Clone)] +pub struct ReadyToken { + pub ts: ZFTimestamp, + pub data: TokenData, + pub action: TokenAction, +} + +#[derive(Debug, Clone)] +pub enum Token { + NotReady(NotReadyToken), + Ready(ReadyToken), +} + +impl Token { + pub fn new_ready(ts: ZFTimestamp, data: TokenData) -> Self { + Self::Ready(ReadyToken { + ts, + data, + action: TokenAction::Consume, + }) + } + + pub fn new_not_ready(ts: ZFTimestamp) -> Self { + Self::NotReady(NotReadyToken { ts }) + } + + pub fn is_ready(&self) -> bool { + match self { + Self::Ready(_) => true, + _ => false, + } + } + + pub fn is_not_ready(&self) -> bool { + match self { + Self::NotReady(_) => true, + _ => false, + } + } + + pub fn consume(&mut self) -> ZFResult<()> { + match self { + Self::Ready(ref mut ready) => { + ready.action = TokenAction::Consume; + Ok(()) + } + _ => Err(ZFError::GenericError), + } + } + + pub fn drop(&mut self) -> ZFResult<()> { + match self { + Self::Ready(ref mut ready) => { + ready.action = TokenAction::Drop; + Ok(()) + } + _ => Err(ZFError::GenericError), + } + } + + pub fn keep_run(&mut self) -> ZFResult<()> { + match self { + Self::Ready(ref mut ready) => { + ready.action = TokenAction::KeepRun; + Ok(()) + } + _ => Err(ZFError::GenericError), + } + } + + pub fn keep(&mut self) -> ZFResult<()> { + match self { + Self::Ready(ref mut ready) => { + ready.action = TokenAction::Keep; + Ok(()) + } + _ => Err(ZFError::GenericError), + } + } + + pub fn data(&self) -> ZFResult { + match self { + Self::Ready(ready) => Ok(ready.data.clone()), + _ => Err(ZFError::GenericError), + } + } + + pub fn action<'a>(&'a self) -> &'a TokenAction { + match self { + Self::Ready(ready) => &ready.action, + Self::NotReady(_) => &TokenAction::Wait, + } + } + + pub fn split(self) -> (Option, TokenAction) { + match self { + Self::Ready(ready) => (Some(ready.data), ready.action), + Self::NotReady(_) => (None, TokenAction::Wait), + } + } +} + +impl From> for Token { + fn from(msg: Arc) -> Self { + match &msg.msg { + Message::Ctrl(_) => Token::NotReady(NotReadyToken { ts: msg.ts }), + Message::Data(data_msg) => match data_msg { + ZFDataMessage::Serialized(ser) => Token::Ready(ReadyToken { + ts: msg.ts, + action: TokenAction::Consume, + data: TokenData::Serialized(ser.clone()), //Use ZBuf + }), + ZFDataMessage::Deserialized(de) => Token::Ready(ReadyToken { + ts: msg.ts, + action: TokenAction::Consume, + data: TokenData::Deserialized(de.clone()), + }), + }, + } + } +} + +#[derive(Debug, Clone)] +pub enum ZFData { + Serialized(Arc>), + Deserialized(Arc), +} + +impl ZFData { + pub fn get_serialized(&self) -> &Arc> { + match self { + Self::Serialized(ser) => ser, + Self::Deserialized(_) => panic!(), + } + } + + pub fn get_deserialized(&self) -> &Arc { + match self { + Self::Deserialized(de) => de, + Self::Serialized(_) => panic!(), + } + } +} + +impl From for ZFData { + fn from(d: TokenData) -> Self { + match d { + TokenData::Serialized(ser) => ZFData::Serialized(ser), + TokenData::Deserialized(de) => ZFData::Deserialized(de), + } + } +} + +#[derive(Debug, Clone)] +pub struct ZFInput(HashMap); + +impl ZFInput { + pub fn new() -> Self { + Self(HashMap::new()) + } + + pub fn insert(&mut self, id: ZFLinkId, data: ZFData) -> Option { + self.0.insert(id, data) + } + + pub fn get(&self, id: &ZFLinkId) -> Option<&ZFData> { + self.0.get(id) + } + + pub fn get_mut(&mut self, id: &ZFLinkId) -> Option<&mut ZFData> { + self.0.get_mut(id) + } +} + +impl<'a> IntoIterator for &'a ZFInput { + type Item = (&'a ZFLinkId, &'a ZFData); + type IntoIter = std::collections::hash_map::Iter<'a, ZFLinkId, ZFData>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EmptyState; + +#[typetag::serde] +impl StateTrait for EmptyState { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_mut_any(&mut self) -> &mut dyn std::any::Any { + self + } +} + +pub fn default_output_rule( + _ctx: ZFContext, + outputs: HashMap>, +) -> OutputRuleResult { + let mut results = HashMap::new(); + for (k, v) in outputs { + // should be ZFMessage::from_data + results.insert(k, Arc::new(ZFMessage::from_data(v))); + } + Ok(results) +} + +pub fn default_input_rule( + _ctx: ZFContext, + inputs: &mut HashMap, +) -> InputRuleResult { + for token in inputs.values() { + match token { + Token::Ready(_) => continue, + Token::NotReady(_) => return Ok(false), + } + } + + Ok(true) +} diff --git a/zenoh-flow/tests/link.rs b/zenoh-flow/tests/link.rs new file mode 100644 index 00000000..f1866eaf --- /dev/null +++ b/zenoh-flow/tests/link.rs @@ -0,0 +1,115 @@ +// +// Copyright (c) 2017, 2021 ADLINK Technology Inc. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0 +// which is available at https://www.apache.org/licenses/LICENSE-2.0. +// +// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 +// +// Contributors: +// ADLINK zenoh team, +// + +use zenoh_flow::async_std::sync::Arc; +use zenoh_flow::runtime::graph::link::{link, ZFLinkReceiver, ZFLinkSender}; +use zenoh_flow::{ZFError, ZFResult}; + +async fn same_task_simple() { + let size = 2; + let id = String::from("0"); + let (sender, mut receiver) = link::(Some(size), id); + + let mut d: u8 = 0; + // Add the first element + let res = sender.send(Arc::new(d)).await; + assert_eq!(res, Ok(())); + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(0u8)))); + + // Add the second element + d = d + 1; + let res = sender.send(Arc::new(d)).await; + assert_eq!(res, Ok(())); + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(1u8)))); +} + +async fn recv_task_simple(mut receiver: ZFLinkReceiver) { + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(0u8)))); + + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(1u8)))); + + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(2u8)))); +} + +async fn send_task_simple(sender: ZFLinkSender) { + let mut d: u8 = 0; + // Add the first element + let res = sender.send(Arc::new(d)).await; + assert_eq!(res, Ok(())); + // Add the second element + d = d + 1; + let res = sender.send(Arc::new(d)).await; + assert_eq!(res, Ok(())); + + // Add the 3rd element + d = d + 1; + let res = sender.send(Arc::new(d)).await; + assert_eq!(res, Ok(())); +} + +async fn recv_task_more(mut receiver: ZFLinkReceiver) { + for n in 0u8..255u8 { + let res = receiver.recv().await; + assert_eq!(res, Ok((String::from("0"), Arc::new(n)))); + } +} + +async fn send_task_more(sender: ZFLinkSender) { + for n in 0u8..255u8 { + let res = sender.send(Arc::new(n)).await; + assert_eq!(res, Ok(())); + } +} + +#[test] +fn ordered_fifo_simple_async() { + async_std::task::block_on(async move { same_task_simple().await }) +} + +#[test] +fn ordered_fifo_simple_two_task_async() { + let size = 2; + let id = String::from("0"); + let (sender, receiver) = link::(Some(size), id); + + let h1 = async_std::task::spawn(async move { send_task_simple(sender).await }); + + let h2 = async_std::task::spawn(async move { recv_task_simple(receiver).await }); + + async_std::task::block_on(async move { + h1.await; + h2.await; + }) +} + +#[test] +fn ordered_fifo_more_two_task_async() { + let size = 20; + let id = String::from("0"); + let (sender, receiver) = link::(Some(size), id); + + let h1 = async_std::task::spawn(async move { send_task_more(sender).await }); + + let h2 = async_std::task::spawn(async move { recv_task_more(receiver).await }); + + async_std::task::block_on(async move { + h1.await; + h2.await; + }) +}