From 9ec4d71f489a9f35f3189f37e080cec66ed59a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 7 Jul 2018 19:36:26 +0200 Subject: [PATCH 01/18] RF [server] create stdout/stderr pipes directly from std::fs::File --- rain_server/src/start/process.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/rain_server/src/start/process.rs b/rain_server/src/start/process.rs index fbde693..8a1b4de 100644 --- a/rain_server/src/start/process.rs +++ b/rain_server/src/start/process.rs @@ -1,4 +1,3 @@ -use std::os::unix::io::{FromRawFd, IntoRawFd}; use std::path::Path; use std::process::{Child, Command, Stdio}; @@ -30,16 +29,12 @@ impl Process { ready: Readiness, command: &mut Command, ) -> Result { - let log_path_out_id = - ::std::fs::File::create(log_dir.join(&format!("{}.out", name)))?.into_raw_fd(); - let log_path_err_id = - ::std::fs::File::create(log_dir.join(&format!("{}.err", name)))?.into_raw_fd(); - - let log_path_out_pipe = unsafe { Stdio::from_raw_fd(log_path_out_id) }; - let log_path_err_pipe = unsafe { Stdio::from_raw_fd(log_path_err_id) }; - - command.stdout(log_path_out_pipe); - command.stderr(log_path_err_pipe); + command.stdout(::std::fs::File::create( + log_dir.join(&format!("{}.out", name)), + )?); + command.stderr(::std::fs::File::create( + log_dir.join(&format!("{}.err", name)), + )?); Ok(Self { name: name.to_string(), From 6578cd3c914c975405bfac63eef23723a7c73670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 25 Apr 2018 00:40:30 +0200 Subject: [PATCH 02/18] [api] add basic Rust client interface --- src/client/client.rs | 62 +++++++++++++++++++++++++++++++++++++++++++ src/client/mod.rs | 2 ++ src/client/session.rs | 3 +++ 3 files changed, 67 insertions(+) create mode 100644 src/client/client.rs create mode 100644 src/client/mod.rs create mode 100644 src/client/session.rs diff --git a/src/client/client.rs b/src/client/client.rs new file mode 100644 index 0000000..45bb1ff --- /dev/null +++ b/src/client/client.rs @@ -0,0 +1,62 @@ +use capnp::capability::Promise; +use capnp_rpc::rpc_twoparty_capnp; +use common::rpc::new_rpc_system; +use futures::Future; +use std::error::Error; +use std::net::SocketAddr; +use tokio_core::net::TcpStream; +use tokio_core::reactor::Core; + +use CLIENT_PROTOCOL_VERSION; +use super::session::Session; + +pub struct Client { + core: Core, + service: ::client_capnp::client_service::Client, +} + +impl Client { + pub fn new(scheduler: &SocketAddr) -> Result> { + let mut core = Core::new()?; + let handle = core.handle(); + let stream = core.run(TcpStream::connect(&scheduler, &handle))?; + stream.set_nodelay(true)?; + + debug!("Connection to server {} established", scheduler); + + let mut rpc = Box::new(new_rpc_system(stream, None)); + let bootstrap: ::server_capnp::server_bootstrap::Client = + rpc.bootstrap(rpc_twoparty_capnp::Side::Server); + handle.spawn(rpc.map_err(|err| panic!("RPC error: {}", err))); + + let mut request = bootstrap.register_as_client_request(); + request.get().set_version(CLIENT_PROTOCOL_VERSION); + + let service = core.run( + request + .send() + .promise + .and_then(|response| Promise::ok(pry!(response.get()).get_service())), + )??; + + Ok(Client { core, service }) + } + + pub fn new_session(&mut self) -> Result> { + let id: i32 = self.core.run( + self.service + .new_session_request() + .send() + .promise + .and_then(|response| Promise::ok(pry!(response.get()).get_session_id())), + )?; + + Ok(Session { id }) + } + + pub fn terminate_server(&mut self) -> Result<(), Box> { + self.core + .run(self.service.terminate_server_request().send().promise)?; + Ok(()) + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs new file mode 100644 index 0000000..cf34e8d --- /dev/null +++ b/src/client/mod.rs @@ -0,0 +1,2 @@ +pub mod client; +pub mod session; diff --git a/src/client/session.rs b/src/client/session.rs new file mode 100644 index 0000000..1bc8dab --- /dev/null +++ b/src/client/session.rs @@ -0,0 +1,3 @@ +pub struct Session { + pub id: i32, +} From dad295cb354a168a78655b341e18bd035b2da740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 25 Apr 2018 00:44:50 +0200 Subject: [PATCH 03/18] [cmd] add server stop command --- rain_server/src/main.rs | 63 +++++++++++++++++++++------- rain_server/src/server/rpc/client.rs | 9 ++++ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/rain_server/src/main.rs b/rain_server/src/main.rs index 3904dd6..b9257c2 100644 --- a/rain_server/src/main.rs +++ b/rain_server/src/main.rs @@ -191,6 +191,22 @@ impl GovernorConfig { } } +fn resolve_server_address(address: &str) -> SocketAddr { + match address.to_socket_addrs() { + Err(_) => { + error!("Cannot resolve server address"); + exit(1); + } + Ok(mut addrs) => match addrs.next() { + None => { + error!("Cannot resolve server address"); + exit(1); + } + Some(ref addr) => *addr, + }, + } +} + fn run_governor(_global_args: &ArgMatches, cmd_args: &ArgMatches) { info!("Starting Rain {} governor", VERSION); let ready_file = cmd_args.value_of("READY_FILE"); @@ -202,19 +218,7 @@ fn run_governor(_global_args: &ArgMatches, cmd_args: &ArgMatches) { server_address = format!("{}:{}", server_address, DEFAULT_SERVER_PORT); } - let server_addr = match server_address.to_socket_addrs() { - Err(_) => { - error!("Cannot resolve server address: "); - exit(1); - } - Ok(mut addrs) => match addrs.next() { - None => { - error!("Cannot resolve server address"); - exit(1); - } - Some(ref addr) => *addr, - }, - }; + let server_addr = resolve_server_address(&server_address); let state = { let config = cmd_args.value_of("GOVERNOR_CONFIG").map(|path| { @@ -415,6 +419,30 @@ fn run_starter(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } } +fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { + let default_address = format!("localhost:{}", DEFAULT_SERVER_PORT); + let mut address = cmd_args + .value_of("SERVER_ADDRESS") + .unwrap_or(&default_address) + .to_string(); + + if !address.contains(':') { + address = format!("{}:{}", address, DEFAULT_SERVER_PORT); + } + + let scheduler: SocketAddr = resolve_server_address(&address); + let mut client = Client::new(&scheduler).unwrap_or_else(|err| { + error!("Couldn't connect to server at {}: {}", address, err); + exit(1); + }); + client.terminate_server().unwrap_or_else(|err| { + error!("Couldn't stop server: {}", err); + exit(1); + }); + + println!("Server at {} was successfully stopped", address); +} + fn init_log() { // T emporary simple logger for better module log control, default level is INFO // TODO: replace with Fern or log4rs later @@ -577,15 +605,22 @@ fn main() { .long("--logdir") .help("Logging directory for governors & server (default /tmp/rain-logs/run-$HOSTANE-$PID)") .takes_value(true))) + .subcommand( // ---- STOP ---- + SubCommand::with_name("stop") + .about("Stop server and all workers connected to it") + .arg(Arg::with_name("SERVER_ADDRESS") + .help("Address of the server (default = localhost:7210)") + .takes_value(true))) .get_matches(); match args.subcommand() { ("server", Some(cmd_args)) => run_server(&args, cmd_args), ("governor", Some(cmd_args)) => run_governor(&args, cmd_args), ("start", Some(cmd_args)) => run_starter(&args, cmd_args), + ("stop", Some(cmd_args)) => stop_server(&args, cmd_args), _ => { error!("No subcommand provided."); - ::std::process::exit(1); + exit(1); } } } diff --git a/rain_server/src/server/rpc/client.rs b/rain_server/src/server/rpc/client.rs index bad6331..a2372e5 100644 --- a/rain_server/src/server/rpc/client.rs +++ b/rain_server/src/server/rpc/client.rs @@ -489,4 +489,13 @@ impl client_service::Server for ClientServiceImpl { results.get_state().unwrap().set_ok(()); Promise::ok(()) } + + fn terminate_server( + &mut self, + params: client_service::TerminateServerParams, + results: client_service::TerminateServerResults, + ) -> Promise<(), ::capnp::Error> { + exit(0); + Promise::ok(()) + } } From c6fca3224ea267ee783325e1d4b9913c7e90bb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 26 Apr 2018 10:02:17 +0200 Subject: [PATCH 04/18] [api] use shared communicator for Rust client, add session close --- rain_server/src/main.rs | 2 +- rain_server/src/server/rpc/client.rs | 5 +- src/client/client.rs | 56 +++++----------------- src/client/communicator.rs | 69 ++++++++++++++++++++++++++++ src/client/mod.rs | 2 + src/client/session.rs | 22 ++++++++- 6 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 src/client/communicator.rs diff --git a/rain_server/src/main.rs b/rain_server/src/main.rs index b9257c2..a9b6187 100644 --- a/rain_server/src/main.rs +++ b/rain_server/src/main.rs @@ -431,7 +431,7 @@ fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } let scheduler: SocketAddr = resolve_server_address(&address); - let mut client = Client::new(&scheduler).unwrap_or_else(|err| { + let mut client = Client::new(scheduler).unwrap_or_else(|err| { error!("Couldn't connect to server at {}: {}", address, err); exit(1); }); diff --git a/rain_server/src/server/rpc/client.rs b/rain_server/src/server/rpc/client.rs index a2372e5..631c76d 100644 --- a/rain_server/src/server/rpc/client.rs +++ b/rain_server/src/server/rpc/client.rs @@ -490,10 +490,11 @@ impl client_service::Server for ClientServiceImpl { Promise::ok(()) } + #[allow(unreachable_code)] fn terminate_server( &mut self, - params: client_service::TerminateServerParams, - results: client_service::TerminateServerResults, + _params: client_service::TerminateServerParams, + _results: client_service::TerminateServerResults, ) -> Promise<(), ::capnp::Error> { exit(0); Promise::ok(()) diff --git a/src/client/client.rs b/src/client/client.rs index 45bb1ff..976adc8 100644 --- a/src/client/client.rs +++ b/src/client/client.rs @@ -1,62 +1,28 @@ -use capnp::capability::Promise; -use capnp_rpc::rpc_twoparty_capnp; -use common::rpc::new_rpc_system; -use futures::Future; use std::error::Error; use std::net::SocketAddr; -use tokio_core::net::TcpStream; -use tokio_core::reactor::Core; use CLIENT_PROTOCOL_VERSION; +use common::wrapped::WrappedRcRefCell; use super::session::Session; +use super::communicator::Communicator; pub struct Client { - core: Core, - service: ::client_capnp::client_service::Client, + comm: WrappedRcRefCell, } impl Client { - pub fn new(scheduler: &SocketAddr) -> Result> { - let mut core = Core::new()?; - let handle = core.handle(); - let stream = core.run(TcpStream::connect(&scheduler, &handle))?; - stream.set_nodelay(true)?; + pub fn new(scheduler: SocketAddr) -> Result> { + let comm = WrappedRcRefCell::wrap(Communicator::new(scheduler, CLIENT_PROTOCOL_VERSION)?); - debug!("Connection to server {} established", scheduler); - - let mut rpc = Box::new(new_rpc_system(stream, None)); - let bootstrap: ::server_capnp::server_bootstrap::Client = - rpc.bootstrap(rpc_twoparty_capnp::Side::Server); - handle.spawn(rpc.map_err(|err| panic!("RPC error: {}", err))); - - let mut request = bootstrap.register_as_client_request(); - request.get().set_version(CLIENT_PROTOCOL_VERSION); - - let service = core.run( - request - .send() - .promise - .and_then(|response| Promise::ok(pry!(response.get()).get_service())), - )??; - - Ok(Client { core, service }) + Ok(Client { comm }) } - pub fn new_session(&mut self) -> Result> { - let id: i32 = self.core.run( - self.service - .new_session_request() - .send() - .promise - .and_then(|response| Promise::ok(pry!(response.get()).get_session_id())), - )?; - - Ok(Session { id }) + pub fn new_session(&self) -> Result> { + let session_id = self.comm.get_mut().new_session()?; + Ok(Session::new(session_id, self.comm.clone())) } - pub fn terminate_server(&mut self) -> Result<(), Box> { - self.core - .run(self.service.terminate_server_request().send().promise)?; - Ok(()) + pub fn terminate_server(&self) -> Result<(), Box> { + self.comm.get_mut().terminate_server() } } diff --git a/src/client/communicator.rs b/src/client/communicator.rs new file mode 100644 index 0000000..1408998 --- /dev/null +++ b/src/client/communicator.rs @@ -0,0 +1,69 @@ +use tokio_core::reactor::Core; +use std::net::SocketAddr; +use tokio_core::net::TcpStream; +use std::error::Error; +use common::rpc::new_rpc_system; +use capnp::capability::Promise; +use capnp_rpc::rpc_twoparty_capnp; +use futures::Future; + +pub struct Communicator { + core: Core, + service: ::client_capnp::client_service::Client, +} + +impl Communicator { + pub fn new(scheduler: SocketAddr, version: i32) -> Result> { + let mut core = Core::new()?; + let handle = core.handle(); + let stream = core.run(TcpStream::connect(&scheduler, &handle))?; + stream.set_nodelay(true)?; + + debug!("Connection to server {} established", scheduler); + + let mut rpc = Box::new(new_rpc_system(stream, None)); + let bootstrap: ::server_capnp::server_bootstrap::Client = + rpc.bootstrap(rpc_twoparty_capnp::Side::Server); + handle.spawn(rpc.map_err(|err| panic!("RPC error: {}", err))); + + let mut request = bootstrap.register_as_client_request(); + request.get().set_version(version); + + let service = core.run( + request + .send() + .promise + .and_then(|response| Promise::ok(pry!(response.get()).get_service())), + )??; + + Ok(Self { core, service }) + } + + pub fn new_session(&mut self) -> Result> { + let id: i32 = self.core.run( + self.service + .new_session_request() + .send() + .promise + .and_then(|response| Promise::ok(pry!(response.get()).get_session_id())), + )?; + + Ok(id) + } + + pub fn close_session(&mut self, id: i32) -> Result> { + self.core.run({ + let mut req = self.service.close_session_request(); + req.get().set_session_id(id); + req.send().promise + })?; + + Ok(true) + } + + pub fn terminate_server(&mut self) -> Result<(), Box> { + self.core + .run(self.service.terminate_server_request().send().promise)?; + Ok(()) + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs index cf34e8d..00801c5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,2 +1,4 @@ pub mod client; pub mod session; + +mod communicator; diff --git a/src/client/session.rs b/src/client/session.rs index 1bc8dab..fcb3ad1 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -1,3 +1,23 @@ +use common::wrapped::WrappedRcRefCell; + +use super::communicator::Communicator; + pub struct Session { - pub id: i32, + id: i32, + comm: WrappedRcRefCell, +} + +impl Session { + pub fn new(id: i32, comm: WrappedRcRefCell) -> Self { + debug!("Session {} created", id); + + Session { id, comm } + } +} + +impl Drop for Session { + fn drop(&mut self) { + self.comm.get_mut().close_session(self.id).unwrap(); + debug!("Session {} destroyed", self.id); + } } From 0f1348c0655af51cc4df7e86bf8e2b03269cce84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 14:54:11 +0200 Subject: [PATCH 05/18] [api] add tasks, data objects and fetch to Rust client --- rain_server/src/main.rs | 2 +- src/client/capnp.rs | 119 +++++++++++++++++++++++++++++++++++++ src/client/communicator.rs | 63 +++++++++++++++++++- src/client/data_object.rs | 16 +++++ src/client/mod.rs | 7 +++ src/client/session.rs | 115 ++++++++++++++++++++++++++++++++++- src/client/task.rs | 50 ++++++++++++++++ 7 files changed, 366 insertions(+), 6 deletions(-) create mode 100644 src/client/capnp.rs create mode 100644 src/client/data_object.rs create mode 100644 src/client/task.rs diff --git a/rain_server/src/main.rs b/rain_server/src/main.rs index a9b6187..b3b8535 100644 --- a/rain_server/src/main.rs +++ b/rain_server/src/main.rs @@ -431,7 +431,7 @@ fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } let scheduler: SocketAddr = resolve_server_address(&address); - let mut client = Client::new(scheduler).unwrap_or_else(|err| { + let client = Client::new(scheduler).unwrap_or_else(|err| { error!("Couldn't connect to server at {}: {}", address, err); exit(1); }); diff --git a/src/client/capnp.rs b/src/client/capnp.rs new file mode 100644 index 0000000..505b770 --- /dev/null +++ b/src/client/capnp.rs @@ -0,0 +1,119 @@ +use super::task::Task; +use super::session::ObjectId; +use capnp::Error; +use client::data_object::DataObject; +use client::task::TaskInput; +use common::Attributes; + +pub trait Serializable<'a> { + type Builder; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box>; +} + +pub struct TaskId(ObjectId); +pub struct DataObjectId(ObjectId); + +impl From for TaskId { + fn from(id: ObjectId) -> Self { + TaskId(id) + } +} +impl From for DataObjectId { + fn from(id: ObjectId) -> Self { + DataObjectId(id) + } +} + +impl<'a> Serializable<'a> for TaskId { + type Builder = ::common_capnp::task_id::Builder<'a>; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + builder.set_id(self.0.id); + builder.set_session_id(self.0.session_id); + + Ok(()) + } +} +impl<'a> Serializable<'a> for DataObjectId { + type Builder = ::common_capnp::data_object_id::Builder<'a>; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + builder.set_id(self.0.id); + builder.set_session_id(self.0.session_id); + + Ok(()) + } +} +impl<'a> Serializable<'a> for TaskInput { + type Builder = ::client_capnp::task::in_data_object::Builder<'a>; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + let id: DataObjectId = self.data_object.get().id.into(); + id.serialize(&mut builder.reborrow().get_id()?)?; + + if let &Some(ref label) = &self.label { + builder.reborrow().set_label(label); + } + + Ok(()) + } +} + +impl<'a> Serializable<'a> for Attributes { + type Builder = ::common_capnp::attributes::Builder<'a>; + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + Ok(self.to_capnp(builder)) + } +} + +impl<'a> Serializable<'a> for Task { + type Builder = ::client_capnp::task::Builder<'a>; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + let task_id: TaskId = self.id.into(); + task_id.serialize(&mut builder.reborrow().get_id()?)?; + builder.set_task_type(self.command.get_task_type()); + + { + let mut inputs_builder = builder.reborrow().init_inputs(self.inputs.len() as u32); + for (i, input) in self.inputs.iter().enumerate() { + input.serialize(&mut inputs_builder.reborrow().get(i as u32))?; + } + } + { + let mut outputs_builder = builder.reborrow().init_outputs(self.outputs.len() as u32); + for (i, output) in self.outputs.iter().enumerate() { + let id: DataObjectId = output.get().id.into(); + id.serialize(&mut outputs_builder.reborrow().get(i as u32))?; + } + } + self.attributes + .serialize(&mut builder.reborrow().get_attributes()?)?; + + Ok(()) + } +} +impl<'a> Serializable<'a> for DataObject { + type Builder = ::client_capnp::data_object::Builder<'a>; + + fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { + let data_obj_id: DataObjectId = self.id.into(); + data_obj_id.serialize(&mut builder.reborrow().get_id()?)?; + builder.set_keep(self.keep); + builder.set_label(&self.label); + builder.set_data_type(::common_capnp::DataType::Blob); // TODO + + if let &Some(ref data) = &self.data { + builder.set_data(&data); + builder.set_has_data(true); + } else { + builder.set_has_data(false); + } + + self.attributes + .serialize(&mut builder.reborrow().get_attributes()?)?; + + Ok(()) + } +} diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 1408998..ba01456 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -7,6 +7,13 @@ use capnp::capability::Promise; use capnp_rpc::rpc_twoparty_capnp; use futures::Future; +use super::task::Task; +use client::data_object::DataObject; +use client::capnp::Serializable; +use common::wrapped::WrappedRcRefCell; +use client::session::ObjectId; +use client::capnp::DataObjectId; + pub struct Communicator { core: Core, service: ::client_capnp::client_service::Client, @@ -50,15 +57,65 @@ impl Communicator { Ok(id) } - - pub fn close_session(&mut self, id: i32) -> Result> { + pub fn close_session(&mut self, id: i32) -> Result<(), Box> { self.core.run({ let mut req = self.service.close_session_request(); req.get().set_session_id(id); req.send().promise })?; - Ok(true) + Ok(()) + } + + pub fn submit( + &mut self, + tasks: &[WrappedRcRefCell], + data_objects: &[WrappedRcRefCell], + ) -> Result<(), Box> { + let mut req = self.service.submit_request(); + { + let mut tasks_builder = req.get().init_tasks(tasks.len() as u32); + for (i, task) in tasks.iter().enumerate() { + task.get() + .serialize(&mut tasks_builder.reborrow().get(i as u32))?; + } + } + { + let mut objects_builder = req.get().init_objects(data_objects.len() as u32); + for (i, obj) in data_objects.iter().enumerate() { + obj.get() + .serialize(&mut objects_builder.reborrow().get(i as u32))?; + } + } + + self.core.run(req.send().promise)?; + + Ok(()) + } + + pub fn fetch(&mut self, object: ObjectId) -> Result, Box> { + let mut req = self.service.fetch_request(); + let obj_id: DataObjectId = object.into(); + obj_id.serialize(&mut req.get().get_id()?)?; + req.get().set_size(1024); + + let response = self.core.run( + req.send() + .promise + .and_then(|response| Promise::ok(response)), + )?; + + let reader = response.get()?; + match reader.get_status().which()? { + ::common_capnp::fetch_result::status::Ok(()) => { + println!("Status: ok"); + } + _ => { + println!("Status: not ok"); + } + } + let data = reader.get_data()?; + Ok(Vec::from(data)) } pub fn terminate_server(&mut self) -> Result<(), Box> { diff --git a/src/client/data_object.rs b/src/client/data_object.rs new file mode 100644 index 0000000..882a57e --- /dev/null +++ b/src/client/data_object.rs @@ -0,0 +1,16 @@ +use client::session::ObjectId; +use common::Attributes; + +pub struct DataObject { + pub id: ObjectId, + pub label: String, + pub keep: bool, + pub data: Option>, + pub attributes: Attributes, +} + +impl DataObject { + pub fn keep(&mut self) { + self.keep = true; + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs index 00801c5..2de80f7 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,4 +1,11 @@ +use client::client::Client; +use std::net::SocketAddr; +use std::net::IpAddr; + pub mod client; pub mod session; mod communicator; +mod task; +mod data_object; +mod capnp; diff --git a/src/client/session.rs b/src/client/session.rs index fcb3ad1..181d1aa 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -1,17 +1,128 @@ use common::wrapped::WrappedRcRefCell; use super::communicator::Communicator; +use client::data_object::DataObject; +use client::task::Task; +use client::task::TaskCommand; +use std::collections::HashMap; +use client::task::ConcatTaskParams; +use std::error::Error; +use client::task::TaskInput; +use common::Attributes; + +#[derive(Copy, Clone, Debug)] +pub struct ObjectId { + pub id: i32, + pub session_id: i32, +} + +impl ObjectId { + pub fn new(id: i32, session_id: i32) -> Self { + ObjectId { id, session_id } + } +} pub struct Session { - id: i32, + pub id: i32, comm: WrappedRcRefCell, + tasks: Vec>, + data_objects: Vec>, + id_counter: i32, } impl Session { pub fn new(id: i32, comm: WrappedRcRefCell) -> Self { debug!("Session {} created", id); - Session { id, comm } + Session { + id, + comm, + tasks: vec![], + data_objects: vec![], + id_counter: 0, + } + } + + pub fn submit(&mut self) -> Result<(), Box> { + self.comm.get_mut().submit(&self.tasks, &self.data_objects)?; + self.tasks.clear(); + self.data_objects.clear(); + + Ok(()) + } + + pub fn fetch(&mut self, object: &DataObject) -> Result, Box> { + self.comm.get_mut().fetch(object.id) + } + + pub fn concat(&mut self, objects: Vec>) -> WrappedRcRefCell { + let inputs = objects + .iter() + .map(|o| TaskInput { + label: None, + data_object: o.clone(), + }) + .collect(); + + let outputs = vec![self.create_object("".to_owned(), None)]; + + self.create_task( + TaskCommand::Concat(ConcatTaskParams { objects }), + inputs, + outputs, + ) + } + + pub fn blob(&mut self, data: Vec) -> WrappedRcRefCell { + self.create_object("".to_owned(), Some(data)) + } + + fn create_object( + &mut self, + label: String, + data: Option>, + ) -> WrappedRcRefCell { + let object = DataObject { + id: self.create_id(), + keep: false, + label, + data, + attributes: Attributes::new(), + }; + let rc = WrappedRcRefCell::wrap(object); + self.data_objects.push(rc.clone()); + + rc + } + + fn create_id(&mut self) -> ObjectId { + let id = self.id_counter; + self.id_counter += 1; + + ObjectId::new(id, self.id) + } + fn create_task( + &mut self, + command: TaskCommand, + inputs: Vec, + outputs: Vec>, + ) -> WrappedRcRefCell { + let mut task = Task { + id: self.create_id(), + command, + inputs, + outputs, + attributes: Attributes::new(), + }; + + let mut resources: HashMap = HashMap::new(); + resources.insert("cpus".to_owned(), 1); + task.attributes.set("resources", resources).unwrap(); + + let rc = WrappedRcRefCell::wrap(task); + self.tasks.push(rc.clone()); + + rc } } diff --git a/src/client/task.rs b/src/client/task.rs new file mode 100644 index 0000000..79bb27f --- /dev/null +++ b/src/client/task.rs @@ -0,0 +1,50 @@ +use client::session::ObjectId; +use client::data_object::DataObject; +use std::error::Error; +use common::wrapped::WrappedRcRefCell; +use common::Attributes; + +pub struct ConcatTaskParams { + pub objects: Vec>, +} + +pub struct OpenTaskParams { + pub filename: String, +} + +pub enum TaskCommand { + Concat(ConcatTaskParams), + Open(OpenTaskParams), +} + +impl TaskCommand { + pub fn get_task_type(&self) -> &'static str { + match self { + &TaskCommand::Concat(_) => "!concat", + &TaskCommand::Open(_) => "!open", + } + } +} + +pub struct TaskInput { + pub label: Option, + pub data_object: WrappedRcRefCell, +} + +pub struct Task { + pub id: ObjectId, + pub command: TaskCommand, + pub inputs: Vec, + pub outputs: Vec>, + pub attributes: Attributes, +} + +impl Task { + pub fn output(&self) -> Result, Box> { + if self.outputs.len() == 1 { + return Ok(self.outputs[0].clone()); + } + + bail!("There is not a single output") + } +} From 630fabc1ff23aa8a72a41e286de3c1320951f91d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 18:34:12 +0200 Subject: [PATCH 06/18] [api] use types from common for Rust client --- src/client/capnp.rs | 87 ++++++-------------------------------- src/client/communicator.rs | 27 +++++------- src/client/data_object.rs | 4 +- src/client/session.rs | 29 ++++++------- src/client/task.rs | 4 +- 5 files changed, 41 insertions(+), 110 deletions(-) diff --git a/src/client/capnp.rs b/src/client/capnp.rs index 505b770..89c6303 100644 --- a/src/client/capnp.rs +++ b/src/client/capnp.rs @@ -1,105 +1,48 @@ use super::task::Task; -use super::session::ObjectId; -use capnp::Error; use client::data_object::DataObject; use client::task::TaskInput; -use common::Attributes; +use common::convert::ToCapnp; -pub trait Serializable<'a> { - type Builder; - - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box>; -} - -pub struct TaskId(ObjectId); -pub struct DataObjectId(ObjectId); - -impl From for TaskId { - fn from(id: ObjectId) -> Self { - TaskId(id) - } -} -impl From for DataObjectId { - fn from(id: ObjectId) -> Self { - DataObjectId(id) - } -} - -impl<'a> Serializable<'a> for TaskId { - type Builder = ::common_capnp::task_id::Builder<'a>; - - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - builder.set_id(self.0.id); - builder.set_session_id(self.0.session_id); - - Ok(()) - } -} -impl<'a> Serializable<'a> for DataObjectId { - type Builder = ::common_capnp::data_object_id::Builder<'a>; - - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - builder.set_id(self.0.id); - builder.set_session_id(self.0.session_id); - - Ok(()) - } -} -impl<'a> Serializable<'a> for TaskInput { +impl<'a> ToCapnp<'a> for TaskInput { type Builder = ::client_capnp::task::in_data_object::Builder<'a>; - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - let id: DataObjectId = self.data_object.get().id.into(); - id.serialize(&mut builder.reborrow().get_id()?)?; + fn to_capnp(&self, builder: &mut Self::Builder) { + self.data_object.get().id.to_capnp(&mut builder.reborrow().get_id().unwrap()); if let &Some(ref label) = &self.label { builder.reborrow().set_label(label); } - - Ok(()) } } -impl<'a> Serializable<'a> for Attributes { - type Builder = ::common_capnp::attributes::Builder<'a>; - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - Ok(self.to_capnp(builder)) - } -} - -impl<'a> Serializable<'a> for Task { +impl<'a> ToCapnp<'a> for Task { type Builder = ::client_capnp::task::Builder<'a>; - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - let task_id: TaskId = self.id.into(); - task_id.serialize(&mut builder.reborrow().get_id()?)?; + fn to_capnp(&self, builder: &mut Self::Builder) { + self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); builder.set_task_type(self.command.get_task_type()); { let mut inputs_builder = builder.reborrow().init_inputs(self.inputs.len() as u32); for (i, input) in self.inputs.iter().enumerate() { - input.serialize(&mut inputs_builder.reborrow().get(i as u32))?; + input.to_capnp(&mut inputs_builder.reborrow().get(i as u32)); } } { let mut outputs_builder = builder.reborrow().init_outputs(self.outputs.len() as u32); for (i, output) in self.outputs.iter().enumerate() { - let id: DataObjectId = output.get().id.into(); - id.serialize(&mut outputs_builder.reborrow().get(i as u32))?; + output.get().id.to_capnp(&mut outputs_builder.reborrow().get(i as u32)); } } self.attributes - .serialize(&mut builder.reborrow().get_attributes()?)?; - - Ok(()) + .to_capnp(&mut builder.reborrow().get_attributes().unwrap()); } } -impl<'a> Serializable<'a> for DataObject { +impl<'a> ToCapnp<'a> for DataObject { type Builder = ::client_capnp::data_object::Builder<'a>; - fn serialize(&self, builder: &mut Self::Builder) -> Result<(), Box> { - let data_obj_id: DataObjectId = self.id.into(); - data_obj_id.serialize(&mut builder.reborrow().get_id()?)?; + fn to_capnp(&self, builder: &mut Self::Builder) { + self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); builder.set_keep(self.keep); builder.set_label(&self.label); builder.set_data_type(::common_capnp::DataType::Blob); // TODO @@ -112,8 +55,6 @@ impl<'a> Serializable<'a> for DataObject { } self.attributes - .serialize(&mut builder.reborrow().get_attributes()?)?; - - Ok(()) + .to_capnp(&mut builder.reborrow().get_attributes().unwrap()); } } diff --git a/src/client/communicator.rs b/src/client/communicator.rs index ba01456..f3b5288 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -9,10 +9,9 @@ use futures::Future; use super::task::Task; use client::data_object::DataObject; -use client::capnp::Serializable; use common::wrapped::WrappedRcRefCell; -use client::session::ObjectId; -use client::capnp::DataObjectId; +use common::id::DataObjectId; +use common::convert::ToCapnp; pub struct Communicator { core: Core, @@ -77,14 +76,14 @@ impl Communicator { let mut tasks_builder = req.get().init_tasks(tasks.len() as u32); for (i, task) in tasks.iter().enumerate() { task.get() - .serialize(&mut tasks_builder.reborrow().get(i as u32))?; + .to_capnp(&mut tasks_builder.reborrow().get(i as u32)); } } { let mut objects_builder = req.get().init_objects(data_objects.len() as u32); for (i, obj) in data_objects.iter().enumerate() { obj.get() - .serialize(&mut objects_builder.reborrow().get(i as u32))?; + .to_capnp(&mut objects_builder.reborrow().get(i as u32)); } } @@ -93,29 +92,23 @@ impl Communicator { Ok(()) } - pub fn fetch(&mut self, object: ObjectId) -> Result, Box> { + pub fn fetch(&mut self, object_id: DataObjectId) -> Result, Box> { let mut req = self.service.fetch_request(); - let obj_id: DataObjectId = object.into(); - obj_id.serialize(&mut req.get().get_id()?)?; + object_id.to_capnp(&mut req.get().get_id().unwrap()); req.get().set_size(1024); - let response = self.core.run( - req.send() - .promise - .and_then(|response| Promise::ok(response)), - )?; + let response = self.core.run(req.send().promise)?; let reader = response.get()?; match reader.get_status().which()? { ::common_capnp::fetch_result::status::Ok(()) => { - println!("Status: ok"); + let data = reader.get_data()?; + Ok(Vec::from(data)) } _ => { - println!("Status: not ok"); + bail!("Non-ok status") } } - let data = reader.get_data()?; - Ok(Vec::from(data)) } pub fn terminate_server(&mut self) -> Result<(), Box> { diff --git a/src/client/data_object.rs b/src/client/data_object.rs index 882a57e..3f2e0ef 100644 --- a/src/client/data_object.rs +++ b/src/client/data_object.rs @@ -1,8 +1,8 @@ -use client::session::ObjectId; use common::Attributes; +use common::id::DataObjectId; pub struct DataObject { - pub id: ObjectId, + pub id: DataObjectId, pub label: String, pub keep: bool, pub data: Option>, diff --git a/src/client/session.rs b/src/client/session.rs index 181d1aa..1fd9d22 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -9,18 +9,9 @@ use client::task::ConcatTaskParams; use std::error::Error; use client::task::TaskInput; use common::Attributes; - -#[derive(Copy, Clone, Debug)] -pub struct ObjectId { - pub id: i32, - pub session_id: i32, -} - -impl ObjectId { - pub fn new(id: i32, session_id: i32) -> Self { - ObjectId { id, session_id } - } -} +use common::id::TaskId; +use common::id::DataObjectId; +use common::id::SId; pub struct Session { pub id: i32, @@ -83,7 +74,7 @@ impl Session { data: Option>, ) -> WrappedRcRefCell { let object = DataObject { - id: self.create_id(), + id: self.create_object_id(), keep: false, label, data, @@ -95,11 +86,17 @@ impl Session { rc } - fn create_id(&mut self) -> ObjectId { + fn create_task_id(&mut self) -> TaskId { + let id = self.id_counter; + self.id_counter += 1; + + TaskId::new(self.id, id) + } + fn create_object_id(&mut self) -> DataObjectId { let id = self.id_counter; self.id_counter += 1; - ObjectId::new(id, self.id) + DataObjectId::new(self.id, id) } fn create_task( &mut self, @@ -108,7 +105,7 @@ impl Session { outputs: Vec>, ) -> WrappedRcRefCell { let mut task = Task { - id: self.create_id(), + id: self.create_task_id(), command, inputs, outputs, diff --git a/src/client/task.rs b/src/client/task.rs index 79bb27f..fbfa4b7 100644 --- a/src/client/task.rs +++ b/src/client/task.rs @@ -1,8 +1,8 @@ -use client::session::ObjectId; use client::data_object::DataObject; use std::error::Error; use common::wrapped::WrappedRcRefCell; use common::Attributes; +use common::id::TaskId; pub struct ConcatTaskParams { pub objects: Vec>, @@ -32,7 +32,7 @@ pub struct TaskInput { } pub struct Task { - pub id: ObjectId, + pub id: TaskId, pub command: TaskCommand, pub inputs: Vec, pub outputs: Vec>, From 99bc661ac083192661e48617676914691292e7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 19:12:28 +0200 Subject: [PATCH 07/18] [api] add macro for serializing Capnp lists --- src/client/capnp.rs | 37 ++++++++++++++++++++++++------------- src/client/communicator.rs | 33 ++++++++++++++++----------------- src/client/mod.rs | 2 ++ 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/client/capnp.rs b/src/client/capnp.rs index 89c6303..95f0c4e 100644 --- a/src/client/capnp.rs +++ b/src/client/capnp.rs @@ -2,12 +2,27 @@ use super::task::Task; use client::data_object::DataObject; use client::task::TaskInput; use common::convert::ToCapnp; +use common::id::DataObjectId; + +macro_rules! capnplist { + ($builder:expr, $items:expr, $name:ident) => { + { + let mut builder = $builder.$name($items.len() as u32); + for (i, obj) in $items.iter().enumerate() { + obj.to_capnp(&mut builder.reborrow().get(i as u32)); + } + } + } +} impl<'a> ToCapnp<'a> for TaskInput { type Builder = ::client_capnp::task::in_data_object::Builder<'a>; fn to_capnp(&self, builder: &mut Self::Builder) { - self.data_object.get().id.to_capnp(&mut builder.reborrow().get_id().unwrap()); + self.data_object + .get() + .id + .to_capnp(&mut builder.reborrow().get_id().unwrap()); if let &Some(ref label) = &self.label { builder.reborrow().set_label(label); @@ -22,18 +37,14 @@ impl<'a> ToCapnp<'a> for Task { self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); builder.set_task_type(self.command.get_task_type()); - { - let mut inputs_builder = builder.reborrow().init_inputs(self.inputs.len() as u32); - for (i, input) in self.inputs.iter().enumerate() { - input.to_capnp(&mut inputs_builder.reborrow().get(i as u32)); - } - } - { - let mut outputs_builder = builder.reborrow().init_outputs(self.outputs.len() as u32); - for (i, output) in self.outputs.iter().enumerate() { - output.get().id.to_capnp(&mut outputs_builder.reborrow().get(i as u32)); - } - } + capnplist!(builder.reborrow(), self.inputs, init_inputs); + capnplist!(builder.reborrow(), + self.outputs + .iter() + .map(|o| o.get().id) + .collect::>(), + init_outputs + ); self.attributes .to_capnp(&mut builder.reborrow().get_attributes().unwrap()); } diff --git a/src/client/communicator.rs b/src/client/communicator.rs index f3b5288..752c4fd 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -6,6 +6,7 @@ use common::rpc::new_rpc_system; use capnp::capability::Promise; use capnp_rpc::rpc_twoparty_capnp; use futures::Future; +use std::cell::Ref; use super::task::Task; use client::data_object::DataObject; @@ -72,20 +73,20 @@ impl Communicator { data_objects: &[WrappedRcRefCell], ) -> Result<(), Box> { let mut req = self.service.submit_request(); - { - let mut tasks_builder = req.get().init_tasks(tasks.len() as u32); - for (i, task) in tasks.iter().enumerate() { - task.get() - .to_capnp(&mut tasks_builder.reborrow().get(i as u32)); - } - } - { - let mut objects_builder = req.get().init_objects(data_objects.len() as u32); - for (i, obj) in data_objects.iter().enumerate() { - obj.get() - .to_capnp(&mut objects_builder.reborrow().get(i as u32)); - } - } + + capnplist!( + req.get(), + tasks.iter().map(|t| t.get()).collect::>>(), + init_tasks + ); + capnplist!( + req.get(), + data_objects + .iter() + .map(|o| o.get()) + .collect::>>(), + init_objects + ); self.core.run(req.send().promise)?; @@ -105,9 +106,7 @@ impl Communicator { let data = reader.get_data()?; Ok(Vec::from(data)) } - _ => { - bail!("Non-ok status") - } + _ => bail!("Non-ok status"), } } diff --git a/src/client/mod.rs b/src/client/mod.rs index 2de80f7..a927b9b 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -5,6 +5,8 @@ use std::net::IpAddr; pub mod client; pub mod session; +#[macro_use] +mod capnp; mod communicator; mod task; mod data_object; From 27d33e3a1887f07f0f1cfc4b4320ff2d92c0367f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 21:12:12 +0200 Subject: [PATCH 08/18] [api] add unkeep, wait and waitSome to Rust client --- src/client/communicator.rs | 69 +++++++++++++++++++- src/client/{data_object.rs => dataobject.rs} | 2 + src/client/mod.rs | 3 +- src/client/{capnp.rs => rpc.rs} | 7 +- src/client/session.rs | 48 +++++++++++++- src/client/task.rs | 2 +- 6 files changed, 120 insertions(+), 11 deletions(-) rename src/client/{data_object.rs => dataobject.rs} (85%) rename src/client/{capnp.rs => rpc.rs} (93%) diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 752c4fd..99074c7 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -9,10 +9,10 @@ use futures::Future; use std::cell::Ref; use super::task::Task; -use client::data_object::DataObject; +use client::dataobject::DataObject; use common::wrapped::WrappedRcRefCell; -use common::id::DataObjectId; -use common::convert::ToCapnp; +use common::id::{DataObjectId, TaskId}; +use common::convert::{FromCapnp, ToCapnp}; pub struct Communicator { core: Core, @@ -93,6 +93,69 @@ impl Communicator { Ok(()) } + pub fn unkeep(&mut self, objects: &[WrappedRcRefCell]) -> Result<(), Box> { + let mut req = self.service.unkeep_request(); + capnplist!( + req.get(), + objects + .iter() + .map(|o| o.get().id) + .collect::>(), + init_object_ids + ); + self.core.run(req.send().promise)?; + Ok(()) + } + + pub fn wait( + &mut self, + tasks: &[WrappedRcRefCell], + objects: &[WrappedRcRefCell], + ) -> Result<(), Box> { + let mut req = self.service.wait_request(); + capnplist!( + req.get(), + tasks.iter().map(|t| t.get().id).collect::>(), + init_task_ids + ); + capnplist!( + req.get(), + objects + .iter() + .map(|o| o.get().id) + .collect::>(), + init_object_ids + ); + self.core.run(req.send().promise)?; + Ok(()) + } + pub fn wait_some( + &mut self, + tasks: &[WrappedRcRefCell], + objects: &[WrappedRcRefCell], + ) -> Result<(Vec, Vec), Box> { + let mut req = self.service.wait_some_request(); + capnplist!( + req.get(), + tasks.iter().map(|t| t.get().id).collect::>(), + init_task_ids + ); + capnplist!( + req.get(), + objects + .iter() + .map(|o| o.get().id) + .collect::>(), + init_object_ids + ); + let res = self.core.run(req.send().promise)?; + + Ok(( + res.get()?.get_finished_tasks()?.iter().map(|id| TaskId::from_capnp(&id)).collect(), + res.get()?.get_finished_objects()?.iter().map(|id| DataObjectId::from_capnp(&id)).collect(), + )) + } + pub fn fetch(&mut self, object_id: DataObjectId) -> Result, Box> { let mut req = self.service.fetch_request(); object_id.to_capnp(&mut req.get().get_id().unwrap()); diff --git a/src/client/data_object.rs b/src/client/dataobject.rs similarity index 85% rename from src/client/data_object.rs rename to src/client/dataobject.rs index 3f2e0ef..30b75d9 100644 --- a/src/client/data_object.rs +++ b/src/client/dataobject.rs @@ -1,5 +1,6 @@ use common::Attributes; use common::id::DataObjectId; +use common::DataType; pub struct DataObject { pub id: DataObjectId, @@ -7,6 +8,7 @@ pub struct DataObject { pub keep: bool, pub data: Option>, pub attributes: Attributes, + pub data_type: DataType, } impl DataObject { diff --git a/src/client/mod.rs b/src/client/mod.rs index a927b9b..64619f5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -6,8 +6,9 @@ pub mod client; pub mod session; #[macro_use] -mod capnp; +mod rpc; mod communicator; +mod dataobject; mod task; mod data_object; mod capnp; diff --git a/src/client/capnp.rs b/src/client/rpc.rs similarity index 93% rename from src/client/capnp.rs rename to src/client/rpc.rs index 95f0c4e..330b459 100644 --- a/src/client/capnp.rs +++ b/src/client/rpc.rs @@ -1,5 +1,5 @@ use super::task::Task; -use client::data_object::DataObject; +use client::dataobject::DataObject; use client::task::TaskInput; use common::convert::ToCapnp; use common::id::DataObjectId; @@ -38,7 +38,8 @@ impl<'a> ToCapnp<'a> for Task { builder.set_task_type(self.command.get_task_type()); capnplist!(builder.reborrow(), self.inputs, init_inputs); - capnplist!(builder.reborrow(), + capnplist!( + builder.reborrow(), self.outputs .iter() .map(|o| o.get().id) @@ -56,7 +57,7 @@ impl<'a> ToCapnp<'a> for DataObject { self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); builder.set_keep(self.keep); builder.set_label(&self.label); - builder.set_data_type(::common_capnp::DataType::Blob); // TODO + builder.set_data_type(self.data_type.to_capnp()); if let &Some(ref data) = &self.data { builder.set_data(&data); diff --git a/src/client/session.rs b/src/client/session.rs index 1fd9d22..f244662 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -1,7 +1,7 @@ use common::wrapped::WrappedRcRefCell; use super::communicator::Communicator; -use client::data_object::DataObject; +use client::dataobject::DataObject; use client::task::Task; use client::task::TaskCommand; use std::collections::HashMap; @@ -12,6 +12,7 @@ use common::Attributes; use common::id::TaskId; use common::id::DataObjectId; use common::id::SId; +use common::DataType; pub struct Session { pub id: i32, @@ -42,11 +43,49 @@ impl Session { Ok(()) } + pub fn unkeep(&mut self, objects: &[WrappedRcRefCell]) -> Result<(), Box> { + self.comm.get_mut().unkeep(objects) + } + pub fn wait( + &mut self, + tasks: &[WrappedRcRefCell], + objects: &[WrappedRcRefCell], + ) -> Result<(), Box> { + self.comm.get_mut().wait(tasks, objects) + } + pub fn wait_some( + &mut self, + tasks: &[WrappedRcRefCell], + objects: &[WrappedRcRefCell], + ) -> Result< + ( + Vec>, + Vec>, + ), + Box, + > { + let task_map: HashMap> = + tasks.iter().map(|t| (t.get().id, t.clone())).collect(); + let object_map: HashMap> = + objects.iter().map(|o| (o.get().id, o.clone())).collect(); + + let (task_ids, object_ids) = self.comm.get_mut().wait_some(tasks, objects)?; + + Ok(( + task_ids.iter() + .filter_map(|id| task_map.get(id).map(|t| t.clone())) + .collect(), + object_ids.iter() + .filter_map(|id| object_map.get(id).map(|o| o.clone())) + .collect(), + )) + } + pub fn fetch(&mut self, object: &DataObject) -> Result, Box> { self.comm.get_mut().fetch(object.id) } - pub fn concat(&mut self, objects: Vec>) -> WrappedRcRefCell { + pub fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell { let inputs = objects .iter() .map(|o| TaskInput { @@ -58,7 +97,9 @@ impl Session { let outputs = vec![self.create_object("".to_owned(), None)]; self.create_task( - TaskCommand::Concat(ConcatTaskParams { objects }), + TaskCommand::Concat(ConcatTaskParams { + objects: Vec::from(objects), + }), inputs, outputs, ) @@ -79,6 +120,7 @@ impl Session { label, data, attributes: Attributes::new(), + data_type: DataType::Blob, }; let rc = WrappedRcRefCell::wrap(object); self.data_objects.push(rc.clone()); diff --git a/src/client/task.rs b/src/client/task.rs index fbfa4b7..8387310 100644 --- a/src/client/task.rs +++ b/src/client/task.rs @@ -1,4 +1,4 @@ -use client::data_object::DataObject; +use client::dataobject::DataObject; use std::error::Error; use common::wrapped::WrappedRcRefCell; use common::Attributes; From 00b55dee23e7bb3427664cfe0d4a38c575491c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 21:43:24 +0200 Subject: [PATCH 09/18] [api] extract common tasks into an external trait implemented on Session --- src/client/communicator.rs | 12 +++++-- src/client/mod.rs | 3 +- src/client/rpc.rs | 2 +- src/client/session.rs | 65 +++++++++++++++----------------------- src/client/task.rs | 24 +------------- src/client/tasks.rs | 35 ++++++++++++++++++++ 6 files changed, 73 insertions(+), 68 deletions(-) create mode 100644 src/client/tasks.rs diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 99074c7..8e4a9cb 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -151,8 +151,16 @@ impl Communicator { let res = self.core.run(req.send().promise)?; Ok(( - res.get()?.get_finished_tasks()?.iter().map(|id| TaskId::from_capnp(&id)).collect(), - res.get()?.get_finished_objects()?.iter().map(|id| DataObjectId::from_capnp(&id)).collect(), + res.get()? + .get_finished_tasks()? + .iter() + .map(|id| TaskId::from_capnp(&id)) + .collect(), + res.get()? + .get_finished_objects()? + .iter() + .map(|id| DataObjectId::from_capnp(&id)) + .collect(), )) } diff --git a/src/client/mod.rs b/src/client/mod.rs index 64619f5..2c60059 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -10,5 +10,4 @@ mod rpc; mod communicator; mod dataobject; mod task; -mod data_object; -mod capnp; +mod tasks; diff --git a/src/client/rpc.rs b/src/client/rpc.rs index 330b459..0a2fd1b 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -35,7 +35,7 @@ impl<'a> ToCapnp<'a> for Task { fn to_capnp(&self, builder: &mut Self::Builder) { self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); - builder.set_task_type(self.command.get_task_type()); + builder.set_task_type(&self.command); capnplist!(builder.reborrow(), self.inputs, init_inputs); capnplist!( diff --git a/src/client/session.rs b/src/client/session.rs index f244662..4647a9c 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -3,9 +3,7 @@ use common::wrapped::WrappedRcRefCell; use super::communicator::Communicator; use client::dataobject::DataObject; use client::task::Task; -use client::task::TaskCommand; use std::collections::HashMap; -use client::task::ConcatTaskParams; use std::error::Error; use client::task::TaskInput; use common::Attributes; @@ -72,10 +70,12 @@ impl Session { let (task_ids, object_ids) = self.comm.get_mut().wait_some(tasks, objects)?; Ok(( - task_ids.iter() + task_ids + .iter() .filter_map(|id| task_map.get(id).map(|t| t.clone())) .collect(), - object_ids.iter() + object_ids + .iter() .filter_map(|id| object_map.get(id).map(|o| o.clone())) .collect(), )) @@ -85,31 +85,17 @@ impl Session { self.comm.get_mut().fetch(object.id) } - pub fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell { - let inputs = objects - .iter() - .map(|o| TaskInput { - label: None, - data_object: o.clone(), - }) - .collect(); - - let outputs = vec![self.create_object("".to_owned(), None)]; - - self.create_task( - TaskCommand::Concat(ConcatTaskParams { - objects: Vec::from(objects), - }), - inputs, - outputs, - ) - } - pub fn blob(&mut self, data: Vec) -> WrappedRcRefCell { self.create_object("".to_owned(), Some(data)) } - fn create_object( + pub(crate) fn create_object_id(&mut self) -> DataObjectId { + let id = self.id_counter; + self.id_counter += 1; + + DataObjectId::new(self.id, id) + } + pub(crate) fn create_object( &mut self, label: String, data: Option>, @@ -128,36 +114,35 @@ impl Session { rc } - fn create_task_id(&mut self) -> TaskId { + pub(crate) fn create_task_id(&mut self) -> TaskId { let id = self.id_counter; self.id_counter += 1; TaskId::new(self.id, id) } - fn create_object_id(&mut self) -> DataObjectId { - let id = self.id_counter; - self.id_counter += 1; - - DataObjectId::new(self.id, id) - } - fn create_task( + pub(crate) fn create_task( &mut self, - command: TaskCommand, + command: String, inputs: Vec, outputs: Vec>, + config: HashMap, + cpus: i32, ) -> WrappedRcRefCell { - let mut task = Task { + let mut attributes = Attributes::new(); + attributes.set("config", config).unwrap(); + + let mut resources: HashMap = HashMap::new(); + resources.insert("cpus".to_owned(), cpus); + attributes.set("resources", resources).unwrap(); + + let task = Task { id: self.create_task_id(), command, inputs, outputs, - attributes: Attributes::new(), + attributes, }; - let mut resources: HashMap = HashMap::new(); - resources.insert("cpus".to_owned(), 1); - task.attributes.set("resources", resources).unwrap(); - let rc = WrappedRcRefCell::wrap(task); self.tasks.push(rc.clone()); diff --git a/src/client/task.rs b/src/client/task.rs index 8387310..e85befc 100644 --- a/src/client/task.rs +++ b/src/client/task.rs @@ -4,28 +4,6 @@ use common::wrapped::WrappedRcRefCell; use common::Attributes; use common::id::TaskId; -pub struct ConcatTaskParams { - pub objects: Vec>, -} - -pub struct OpenTaskParams { - pub filename: String, -} - -pub enum TaskCommand { - Concat(ConcatTaskParams), - Open(OpenTaskParams), -} - -impl TaskCommand { - pub fn get_task_type(&self) -> &'static str { - match self { - &TaskCommand::Concat(_) => "!concat", - &TaskCommand::Open(_) => "!open", - } - } -} - pub struct TaskInput { pub label: Option, pub data_object: WrappedRcRefCell, @@ -33,7 +11,7 @@ pub struct TaskInput { pub struct Task { pub id: TaskId, - pub command: TaskCommand, + pub command: String, pub inputs: Vec, pub outputs: Vec>, pub attributes: Attributes, diff --git a/src/client/tasks.rs b/src/client/tasks.rs new file mode 100644 index 0000000..e1615e5 --- /dev/null +++ b/src/client/tasks.rs @@ -0,0 +1,35 @@ +use client::task::TaskInput; +use client::session::Session; +use common::wrapped::WrappedRcRefCell; +use client::dataobject::DataObject; +use client::task::Task; +use std::collections::HashMap; + +pub trait CommonTasks { + fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell; + fn open(&mut self, filename: String) -> WrappedRcRefCell; +} + +impl CommonTasks for Session { + fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell { + let inputs = objects + .iter() + .map(|o| TaskInput { + label: None, + data_object: o.clone(), + }) + .collect(); + + let outputs = vec![self.create_object("".to_owned(), None)]; + + self.create_task("!concat".to_owned(), inputs, outputs, HashMap::new(), 1) + } + fn open(&mut self, filename: String) -> WrappedRcRefCell { + let mut config = HashMap::new(); + config.insert("path".to_owned(), filename); + + let outputs = vec![self.create_object("".to_owned(), None)]; + + self.create_task("!open".to_owned(), vec![], outputs, config, 1) + } +} From 53bee79c4c15e98a735639c10ec328a477e713da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 22:43:52 +0200 Subject: [PATCH 10/18] [api] add export task --- src/client/tasks.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/client/tasks.rs b/src/client/tasks.rs index e1615e5..76f0c19 100644 --- a/src/client/tasks.rs +++ b/src/client/tasks.rs @@ -8,6 +8,11 @@ use std::collections::HashMap; pub trait CommonTasks { fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell; fn open(&mut self, filename: String) -> WrappedRcRefCell; + fn export( + &mut self, + object: WrappedRcRefCell, + filename: String, + ) -> WrappedRcRefCell; } impl CommonTasks for Session { @@ -32,4 +37,19 @@ impl CommonTasks for Session { self.create_task("!open".to_owned(), vec![], outputs, config, 1) } + fn export( + &mut self, + object: WrappedRcRefCell, + filename: String, + ) -> WrappedRcRefCell { + let mut config = HashMap::new(); + config.insert("path".to_owned(), filename); + + let input = TaskInput { + label: None, + data_object: object.clone(), + }; + + self.create_task("!export".to_owned(), vec![input], vec![], config, 1) + } } From 7c5bb939c20865ec1a3ce99a1342748307f58a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 2 May 2018 22:56:39 +0200 Subject: [PATCH 11/18] [api] add waitAll for Rust client --- rain_server/src/server/rpc/client.rs | 2 +- src/client/communicator.rs | 40 +++++----------------------- src/client/mod.rs | 4 --- src/client/session.rs | 36 ++++++++++++++++++------- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/rain_server/src/server/rpc/client.rs b/rain_server/src/server/rpc/client.rs index 631c76d..4f34f10 100644 --- a/rain_server/src/server/rpc/client.rs +++ b/rain_server/src/server/rpc/client.rs @@ -283,7 +283,7 @@ impl client_service::Server for ClientServiceImpl { object_ids.len() ); Promise::err(::capnp::Error::failed( - "wait_sone is not implemented yet".to_string(), + "wait_some is not implemented yet".to_string(), )) } diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 8e4a9cb..5646746 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -107,47 +107,21 @@ impl Communicator { Ok(()) } - pub fn wait( - &mut self, - tasks: &[WrappedRcRefCell], - objects: &[WrappedRcRefCell], - ) -> Result<(), Box> { + pub fn wait(&mut self, tasks: &[TaskId], objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.wait_request(); - capnplist!( - req.get(), - tasks.iter().map(|t| t.get().id).collect::>(), - init_task_ids - ); - capnplist!( - req.get(), - objects - .iter() - .map(|o| o.get().id) - .collect::>(), - init_object_ids - ); + capnplist!(req.get(), tasks, init_task_ids); + capnplist!(req.get(), objects, init_object_ids); self.core.run(req.send().promise)?; Ok(()) } pub fn wait_some( &mut self, - tasks: &[WrappedRcRefCell], - objects: &[WrappedRcRefCell], + tasks: &[TaskId], + objects: &[DataObjectId], ) -> Result<(Vec, Vec), Box> { let mut req = self.service.wait_some_request(); - capnplist!( - req.get(), - tasks.iter().map(|t| t.get().id).collect::>(), - init_task_ids - ); - capnplist!( - req.get(), - objects - .iter() - .map(|o| o.get().id) - .collect::>(), - init_object_ids - ); + capnplist!(req.get(), tasks, init_task_ids); + capnplist!(req.get(), objects, init_object_ids); let res = self.core.run(req.send().promise)?; Ok(( diff --git a/src/client/mod.rs b/src/client/mod.rs index 2c60059..3f1a0aa 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,7 +1,3 @@ -use client::client::Client; -use std::net::SocketAddr; -use std::net::IpAddr; - pub mod client; pub mod session; diff --git a/src/client/session.rs b/src/client/session.rs index 4647a9c..0da688f 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -49,7 +49,13 @@ impl Session { tasks: &[WrappedRcRefCell], objects: &[WrappedRcRefCell], ) -> Result<(), Box> { - self.comm.get_mut().wait(tasks, objects) + self.comm.get_mut().wait( + &tasks.iter().map(|t| t.get().id).collect::>(), + &objects + .iter() + .map(|o| o.get().id) + .collect::>(), + ) } pub fn wait_some( &mut self, @@ -62,24 +68,36 @@ impl Session { ), Box, > { - let task_map: HashMap> = - tasks.iter().map(|t| (t.get().id, t.clone())).collect(); - let object_map: HashMap> = - objects.iter().map(|o| (o.get().id, o.clone())).collect(); - - let (task_ids, object_ids) = self.comm.get_mut().wait_some(tasks, objects)?; + let task_map: HashMap> = + tasks.iter().map(|t| (t.get().id, t)).collect(); + let object_map: HashMap> = + objects.iter().map(|o| (o.get().id, o)).collect(); + + let (task_ids, object_ids) = self.comm.get_mut().wait_some( + &tasks.iter().map(|t| t.get().id).collect::>(), + &objects + .iter() + .map(|o| o.get().id) + .collect::>(), + )?; Ok(( task_ids .iter() - .filter_map(|id| task_map.get(id).map(|t| t.clone())) + .filter_map(|id| task_map.get(id).map(|t| (*t).clone())) .collect(), object_ids .iter() - .filter_map(|id| object_map.get(id).map(|o| o.clone())) + .filter_map(|id| object_map.get(id).map(|o| (*o).clone())) .collect(), )) } + pub fn wait_all(&mut self) -> Result<(), Box> { + self.comm.get_mut().wait( + &vec![TaskId::new(self.id, ::common_capnp::ALL_TASKS_ID)], + &vec![], + ) + } pub fn fetch(&mut self, object: &DataObject) -> Result, Box> { self.comm.get_mut().fetch(object.id) From fee393abbd8493c8d32e8f3c5c971f5c8ca8629b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 3 May 2018 13:07:08 +0200 Subject: [PATCH 12/18] [api] add local cluster --- rain_server/src/main.rs | 10 +++++-- rain_server/src/start/common.rs | 33 +++++++++++++++++++++ rain_server/src/start/mod.rs | 1 + rain_server/src/start/starter.rs | 6 ++++ src/client/mod.rs | 3 ++ src/start/localcluster.rs | 50 ++++++++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 src/start/localcluster.rs diff --git a/rain_server/src/main.rs b/rain_server/src/main.rs index b3b8535..7f96d39 100644 --- a/rain_server/src/main.rs +++ b/rain_server/src/main.rs @@ -12,7 +12,6 @@ extern crate chrono; #[macro_use] extern crate clap; extern crate env_logger; -#[macro_use] extern crate error_chain; extern crate fs_extra; extern crate futures; @@ -50,6 +49,7 @@ mod wrapped; use clap::{App, Arg, ArgMatches, SubCommand}; use nix::unistd::getpid; + use std::collections::HashMap; use std::error::Error; use std::io::Read; @@ -60,6 +60,7 @@ use std::process::exit; use rain_core::sys::{create_ready_file, get_hostname}; use rain_core::{errors::*, utils::*}; +use clap::{App, Arg, ArgMatches, SubCommand}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_SERVER_PORT: u16 = 7210; @@ -140,6 +141,7 @@ fn run_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } } +<<<<<<< 7999023457517e7570c2373ab414486b9fe0163c:rain_server/src/main.rs fn default_working_directory() -> PathBuf { let pid = getpid(); let hostname = get_hostname(); @@ -169,6 +171,8 @@ fn ensure_directory(dir: &Path, name: &str) -> Result<()> { Ok(()) } +======= +>>>>>>> [api] add local cluster:src/bin.rs // TODO: Do some serious configuration file and unify configurations // Right now, it is just a quick hack for supporting executors @@ -431,7 +435,7 @@ fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } let scheduler: SocketAddr = resolve_server_address(&address); - let client = Client::new(scheduler).unwrap_or_else(|err| { + let client = client::Client::new(scheduler).unwrap_or_else(|err| { error!("Couldn't connect to server at {}: {}", address, err); exit(1); }); @@ -444,7 +448,7 @@ fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } fn init_log() { - // T emporary simple logger for better module log control, default level is INFO + // Temporary simple logger for better module log control, default level is INFO // TODO: replace with Fern or log4rs later if std::env::var("RUST_LOG").is_err() { std::env::set_var("RUST_LOG", "info"); diff --git a/rain_server/src/start/common.rs b/rain_server/src/start/common.rs index 10289e0..c96f6c3 100644 --- a/rain_server/src/start/common.rs +++ b/rain_server/src/start/common.rs @@ -1,4 +1,9 @@ use std::path::PathBuf; +use nix::unistd::getpid; +use std::path::Path; +use errors::Result; +use std::fs::create_dir_all; +use std::error::Error; pub enum Readiness { /// Ready file is a file that @@ -6,3 +11,31 @@ pub enum Readiness { WaitingForReadyFile(PathBuf), IsReady, } + +pub fn default_working_directory() -> PathBuf { + let pid = getpid(); + let hostname = ::common::sys::get_hostname(); + PathBuf::from("/tmp/rain-work").join(format!("worker-{}-{}", hostname, pid)) +} + +pub fn default_logging_directory(basename: &str) -> PathBuf { + let pid = getpid(); + let hostname = ::common::sys::get_hostname(); + PathBuf::from("/tmp/rain-logs").join(format!("{}-{}-{}", basename, hostname, pid)) +} +pub fn ensure_directory(dir: &Path, name: &str) -> Result<()> { + if !dir.exists() { + debug!("{} not found, creating ... {:?}", name, dir); + if let Err(e) = create_dir_all(dir) { + bail!(format!( + "{} {:?} cannot by created: {}", + name, + dir, + e.description() + )); + } + } else if !dir.is_dir() { + bail!("{} {:?} exists but it is not a directory", name, dir); + } + Ok(()) +} diff --git a/rain_server/src/start/mod.rs b/rain_server/src/start/mod.rs index 0519e43..0b7d57e 100644 --- a/rain_server/src/start/mod.rs +++ b/rain_server/src/start/mod.rs @@ -1,4 +1,5 @@ pub mod common; +pub mod localcluster; pub mod process; pub mod ssh; pub mod starter; diff --git a/rain_server/src/start/starter.rs b/rain_server/src/start/starter.rs index 75fe49a..3024f72 100644 --- a/rain_server/src/start/starter.rs +++ b/rain_server/src/start/starter.rs @@ -11,6 +11,12 @@ use std::process::Command; use start::common::Readiness; use start::process::Process; use start::ssh::RemoteProcess; +use errors::Result; + +use nix::unistd::getpid; +use std::io::BufReader; +use std::io::BufRead; +use std::fs::File; pub struct StarterConfig { /// Number of local governor that will be spawned diff --git a/src/client/mod.rs b/src/client/mod.rs index 3f1a0aa..c091381 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -7,3 +7,6 @@ mod communicator; mod dataobject; mod task; mod tasks; + +pub use self::client::Client; +pub use self::session::Session; diff --git a/src/start/localcluster.rs b/src/start/localcluster.rs new file mode 100644 index 0000000..1b6381b --- /dev/null +++ b/src/start/localcluster.rs @@ -0,0 +1,50 @@ +use start::starter::{Starter, StarterConfig}; +use std::net::SocketAddr; +use std::error::Error; +use std::iter; +use client::client::Client; +use start::common::{default_logging_directory, ensure_directory}; + +pub struct LocalCluster { + listen_addr: SocketAddr, + starter: Starter, +} + +impl LocalCluster { + pub fn new(worker_count: usize, listen_port: u16, http_port: u16) -> Result> { + let listen_addr = SocketAddr::new("127.0.0.1".parse()?, listen_port); + let http_addr = SocketAddr::new("127.0.0.1".parse()?, http_port); + + let log_dir = default_logging_directory("rain"); + ensure_directory(&log_dir, "logging directory")?; + let workers = iter::repeat(Some(1)).take(worker_count).collect(); + + let config = StarterConfig::new( + workers, + listen_addr, + http_addr, + &log_dir, + "".to_owned(), + false, + vec![], + ); + + let mut cluster = LocalCluster { + listen_addr, + starter: Starter::new(config), + }; + cluster.starter.start()?; + + Ok(cluster) + } + + pub fn create_client(&self) -> Result> { + Client::new(self.listen_addr) + } +} + +impl Drop for LocalCluster { + fn drop(&mut self) { + self.starter.kill_all(); + } +} From 05f439a8f04666a0ad85cba5aac37dd835a602cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 4 May 2018 10:59:25 +0200 Subject: [PATCH 13/18] [api] add simple creation method to LocalCluster --- src/start/localcluster.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/start/localcluster.rs b/src/start/localcluster.rs index 1b6381b..9fc0751 100644 --- a/src/start/localcluster.rs +++ b/src/start/localcluster.rs @@ -1,7 +1,6 @@ use start::starter::{Starter, StarterConfig}; use std::net::SocketAddr; use std::error::Error; -use std::iter; use client::client::Client; use start::common::{default_logging_directory, ensure_directory}; @@ -11,16 +10,19 @@ pub struct LocalCluster { } impl LocalCluster { - pub fn new(worker_count: usize, listen_port: u16, http_port: u16) -> Result> { + pub fn new( + worker_cpus: Vec>, + listen_port: u16, + http_port: u16, + ) -> Result> { let listen_addr = SocketAddr::new("127.0.0.1".parse()?, listen_port); let http_addr = SocketAddr::new("127.0.0.1".parse()?, http_port); let log_dir = default_logging_directory("rain"); ensure_directory(&log_dir, "logging directory")?; - let workers = iter::repeat(Some(1)).take(worker_count).collect(); let config = StarterConfig::new( - workers, + worker_cpus, listen_addr, http_addr, &log_dir, @@ -37,6 +39,9 @@ impl LocalCluster { Ok(cluster) } + pub fn new_simple(listen_port: u16, http_port: u16) -> Result> { + Self::new(vec![None], listen_port, http_port) + } pub fn create_client(&self) -> Result> { Client::new(self.listen_addr) From 1e2f511771e16da895164bf1d233aca59bcfca3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 4 May 2018 12:41:44 +0200 Subject: [PATCH 14/18] [api] use Rc for DataObject in session --- src/client/communicator.rs | 33 +++++----------- src/client/dataobject.rs | 7 ++-- src/client/mod.rs | 2 +- src/client/rpc.rs | 5 +-- src/client/session.rs | 77 +++++++++++++++++--------------------- src/client/task.rs | 9 ++--- src/client/tasks.rs | 28 +++++--------- 7 files changed, 65 insertions(+), 96 deletions(-) diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 5646746..6fbc4f4 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -9,10 +9,9 @@ use futures::Future; use std::cell::Ref; use super::task::Task; -use client::dataobject::DataObject; -use common::wrapped::WrappedRcRefCell; use common::id::{DataObjectId, TaskId}; use common::convert::{FromCapnp, ToCapnp}; +use client::dataobject::DataObject; pub struct Communicator { core: Core, @@ -67,24 +66,19 @@ impl Communicator { Ok(()) } - pub fn submit( - &mut self, - tasks: &[WrappedRcRefCell], - data_objects: &[WrappedRcRefCell], - ) -> Result<(), Box> { + pub fn submit(&mut self, tasks: &[Ref], data_objects: &[D]) -> Result<(), Box> + where + D: AsRef, + { let mut req = self.service.submit_request(); - capnplist!( - req.get(), - tasks.iter().map(|t| t.get()).collect::>>(), - init_tasks - ); + capnplist!(req.get(), tasks, init_tasks); capnplist!( req.get(), data_objects .iter() - .map(|o| o.get()) - .collect::>>(), + .map(|t| t.as_ref()) + .collect::>(), init_objects ); @@ -93,16 +87,9 @@ impl Communicator { Ok(()) } - pub fn unkeep(&mut self, objects: &[WrappedRcRefCell]) -> Result<(), Box> { + pub fn unkeep(&mut self, objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.unkeep_request(); - capnplist!( - req.get(), - objects - .iter() - .map(|o| o.get().id) - .collect::>(), - init_object_ids - ); + capnplist!(req.get(), objects, init_object_ids); self.core.run(req.send().promise)?; Ok(()) } diff --git a/src/client/dataobject.rs b/src/client/dataobject.rs index 30b75d9..0fdc643 100644 --- a/src/client/dataobject.rs +++ b/src/client/dataobject.rs @@ -1,18 +1,19 @@ use common::Attributes; use common::id::DataObjectId; use common::DataType; +use std::cell::Cell; pub struct DataObject { pub id: DataObjectId, pub label: String, - pub keep: bool, + pub keep: Cell, pub data: Option>, pub attributes: Attributes, pub data_type: DataType, } impl DataObject { - pub fn keep(&mut self) { - self.keep = true; + pub fn keep(&self) { + self.keep.set(true); } } diff --git a/src/client/mod.rs b/src/client/mod.rs index c091381..975fed6 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,12 +1,12 @@ pub mod client; pub mod session; +pub mod tasks; #[macro_use] mod rpc; mod communicator; mod dataobject; mod task; -mod tasks; pub use self::client::Client; pub use self::session::Session; diff --git a/src/client/rpc.rs b/src/client/rpc.rs index 0a2fd1b..0784f11 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -20,7 +20,6 @@ impl<'a> ToCapnp<'a> for TaskInput { fn to_capnp(&self, builder: &mut Self::Builder) { self.data_object - .get() .id .to_capnp(&mut builder.reborrow().get_id().unwrap()); @@ -42,7 +41,7 @@ impl<'a> ToCapnp<'a> for Task { builder.reborrow(), self.outputs .iter() - .map(|o| o.get().id) + .map(|o| o.id) .collect::>(), init_outputs ); @@ -55,7 +54,7 @@ impl<'a> ToCapnp<'a> for DataObject { fn to_capnp(&self, builder: &mut Self::Builder) { self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); - builder.set_keep(self.keep); + builder.set_keep(self.keep.get()); builder.set_label(&self.label); builder.set_data_type(self.data_type.to_capnp()); diff --git a/src/client/session.rs b/src/client/session.rs index 0da688f..76f4503 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -11,12 +11,17 @@ use common::id::TaskId; use common::id::DataObjectId; use common::id::SId; use common::DataType; +use std::cell::{Cell, Ref}; +use std::rc::Rc; + +pub type DataObjectPtr = Rc; +pub type TaskPtr = WrappedRcRefCell; pub struct Session { pub id: i32, comm: WrappedRcRefCell, - tasks: Vec>, - data_objects: Vec>, + tasks: Vec, + data_objects: Vec, id_counter: i32, } @@ -34,51 +39,43 @@ impl Session { } pub fn submit(&mut self) -> Result<(), Box> { - self.comm.get_mut().submit(&self.tasks, &self.data_objects)?; + self.comm.get_mut().submit( + &self.tasks + .iter() + .map(|t| t.get()) + .collect::>>(), + &self.data_objects, + )?; self.tasks.clear(); self.data_objects.clear(); Ok(()) } - pub fn unkeep(&mut self, objects: &[WrappedRcRefCell]) -> Result<(), Box> { - self.comm.get_mut().unkeep(objects) + pub fn unkeep(&mut self, objects: &[DataObjectPtr]) -> Result<(), Box> { + self.comm + .get_mut() + .unkeep(&objects.iter().map(|o| o.id).collect::>()) } - pub fn wait( - &mut self, - tasks: &[WrappedRcRefCell], - objects: &[WrappedRcRefCell], - ) -> Result<(), Box> { + + pub fn wait(&mut self, tasks: &[TaskPtr], objects: &[DataObjectPtr]) -> Result<(), Box> { self.comm.get_mut().wait( &tasks.iter().map(|t| t.get().id).collect::>(), - &objects - .iter() - .map(|o| o.get().id) - .collect::>(), + &objects.iter().map(|o| o.id).collect::>(), ) } pub fn wait_some( &mut self, - tasks: &[WrappedRcRefCell], - objects: &[WrappedRcRefCell], - ) -> Result< - ( - Vec>, - Vec>, - ), - Box, - > { - let task_map: HashMap> = - tasks.iter().map(|t| (t.get().id, t)).collect(); - let object_map: HashMap> = - objects.iter().map(|o| (o.get().id, o)).collect(); + tasks: &[TaskPtr], + objects: &[DataObjectPtr], + ) -> Result<(Vec, Vec), Box> { + let task_map: HashMap = tasks.iter().map(|t| (t.get().id, t)).collect(); + let object_map: HashMap = + objects.iter().map(|o| (o.id, o)).collect(); let (task_ids, object_ids) = self.comm.get_mut().wait_some( &tasks.iter().map(|t| t.get().id).collect::>(), - &objects - .iter() - .map(|o| o.get().id) - .collect::>(), + &objects.iter().map(|o| o.id).collect::>(), )?; Ok(( @@ -103,7 +100,7 @@ impl Session { self.comm.get_mut().fetch(object.id) } - pub fn blob(&mut self, data: Vec) -> WrappedRcRefCell { + pub fn blob(&mut self, data: Vec) -> DataObjectPtr { self.create_object("".to_owned(), Some(data)) } @@ -113,20 +110,16 @@ impl Session { DataObjectId::new(self.id, id) } - pub(crate) fn create_object( - &mut self, - label: String, - data: Option>, - ) -> WrappedRcRefCell { + pub(crate) fn create_object(&mut self, label: String, data: Option>) -> DataObjectPtr { let object = DataObject { id: self.create_object_id(), - keep: false, + keep: Cell::new(false), label, data, attributes: Attributes::new(), data_type: DataType::Blob, }; - let rc = WrappedRcRefCell::wrap(object); + let rc = Rc::new(object); self.data_objects.push(rc.clone()); rc @@ -138,14 +131,14 @@ impl Session { TaskId::new(self.id, id) } - pub(crate) fn create_task( + pub fn create_task( &mut self, command: String, inputs: Vec, - outputs: Vec>, + outputs: Vec, config: HashMap, cpus: i32, - ) -> WrappedRcRefCell { + ) -> TaskPtr { let mut attributes = Attributes::new(); attributes.set("config", config).unwrap(); diff --git a/src/client/task.rs b/src/client/task.rs index e85befc..a3342a0 100644 --- a/src/client/task.rs +++ b/src/client/task.rs @@ -1,24 +1,23 @@ -use client::dataobject::DataObject; use std::error::Error; -use common::wrapped::WrappedRcRefCell; use common::Attributes; use common::id::TaskId; +use super::session::DataObjectPtr; pub struct TaskInput { pub label: Option, - pub data_object: WrappedRcRefCell, + pub data_object: DataObjectPtr, } pub struct Task { pub id: TaskId, pub command: String, pub inputs: Vec, - pub outputs: Vec>, + pub outputs: Vec, pub attributes: Attributes, } impl Task { - pub fn output(&self) -> Result, Box> { + pub fn output(&self) -> Result> { if self.outputs.len() == 1 { return Ok(self.outputs[0].clone()); } diff --git a/src/client/tasks.rs b/src/client/tasks.rs index 76f0c19..1926857 100644 --- a/src/client/tasks.rs +++ b/src/client/tasks.rs @@ -1,22 +1,16 @@ -use client::task::TaskInput; -use client::session::Session; -use common::wrapped::WrappedRcRefCell; -use client::dataobject::DataObject; -use client::task::Task; use std::collections::HashMap; +use super::task::TaskInput; +use super::session::{DataObjectPtr, Session, TaskPtr}; + pub trait CommonTasks { - fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell; - fn open(&mut self, filename: String) -> WrappedRcRefCell; - fn export( - &mut self, - object: WrappedRcRefCell, - filename: String, - ) -> WrappedRcRefCell; + fn concat(&mut self, objects: &[DataObjectPtr]) -> TaskPtr; + fn open(&mut self, filename: String) -> TaskPtr; + fn export(&mut self, object: DataObjectPtr, filename: String) -> TaskPtr; } impl CommonTasks for Session { - fn concat(&mut self, objects: &[WrappedRcRefCell]) -> WrappedRcRefCell { + fn concat(&mut self, objects: &[DataObjectPtr]) -> TaskPtr { let inputs = objects .iter() .map(|o| TaskInput { @@ -29,7 +23,7 @@ impl CommonTasks for Session { self.create_task("!concat".to_owned(), inputs, outputs, HashMap::new(), 1) } - fn open(&mut self, filename: String) -> WrappedRcRefCell { + fn open(&mut self, filename: String) -> TaskPtr { let mut config = HashMap::new(); config.insert("path".to_owned(), filename); @@ -37,11 +31,7 @@ impl CommonTasks for Session { self.create_task("!open".to_owned(), vec![], outputs, config, 1) } - fn export( - &mut self, - object: WrappedRcRefCell, - filename: String, - ) -> WrappedRcRefCell { + fn export(&mut self, object: DataObjectPtr, filename: String) -> TaskPtr { let mut config = HashMap::new(); config.insert("path".to_owned(), filename); From 1bab01269dc1801986a4fb6929e2fa22bea0aeb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 4 May 2018 12:51:47 +0200 Subject: [PATCH 15/18] [api] add macro for parsing Capnp lists --- src/client/communicator.rs | 46 ++++++++++++-------------------------- src/client/rpc.rs | 17 +++++++++++--- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 6fbc4f4..6024d20 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -3,7 +3,6 @@ use std::net::SocketAddr; use tokio_core::net::TcpStream; use std::error::Error; use common::rpc::new_rpc_system; -use capnp::capability::Promise; use capnp_rpc::rpc_twoparty_capnp; use futures::Future; use std::cell::Ref; @@ -35,24 +34,16 @@ impl Communicator { let mut request = bootstrap.register_as_client_request(); request.get().set_version(version); - let service = core.run( - request - .send() - .promise - .and_then(|response| Promise::ok(pry!(response.get()).get_service())), - )??; + let service = core.run(request.send().promise)?.get()?.get_service()?; Ok(Self { core, service }) } pub fn new_session(&mut self) -> Result> { - let id: i32 = self.core.run( - self.service - .new_session_request() - .send() - .promise - .and_then(|response| Promise::ok(pry!(response.get()).get_session_id())), - )?; + let id: i32 = self.core + .run(self.service.new_session_request().send().promise)? + .get()? + .get_session_id(); Ok(id) } @@ -72,8 +63,8 @@ impl Communicator { { let mut req = self.service.submit_request(); - capnplist!(req.get(), tasks, init_tasks); - capnplist!( + to_capnp_list!(req.get(), tasks, init_tasks); + to_capnp_list!( req.get(), data_objects .iter() @@ -81,7 +72,6 @@ impl Communicator { .collect::>(), init_objects ); - self.core.run(req.send().promise)?; Ok(()) @@ -89,15 +79,15 @@ impl Communicator { pub fn unkeep(&mut self, objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.unkeep_request(); - capnplist!(req.get(), objects, init_object_ids); + to_capnp_list!(req.get(), objects, init_object_ids); self.core.run(req.send().promise)?; Ok(()) } pub fn wait(&mut self, tasks: &[TaskId], objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.wait_request(); - capnplist!(req.get(), tasks, init_task_ids); - capnplist!(req.get(), objects, init_object_ids); + to_capnp_list!(req.get(), tasks, init_task_ids); + to_capnp_list!(req.get(), objects, init_object_ids); self.core.run(req.send().promise)?; Ok(()) } @@ -107,21 +97,13 @@ impl Communicator { objects: &[DataObjectId], ) -> Result<(Vec, Vec), Box> { let mut req = self.service.wait_some_request(); - capnplist!(req.get(), tasks, init_task_ids); - capnplist!(req.get(), objects, init_object_ids); + to_capnp_list!(req.get(), tasks, init_task_ids); + to_capnp_list!(req.get(), objects, init_object_ids); let res = self.core.run(req.send().promise)?; Ok(( - res.get()? - .get_finished_tasks()? - .iter() - .map(|id| TaskId::from_capnp(&id)) - .collect(), - res.get()? - .get_finished_objects()? - .iter() - .map(|id| DataObjectId::from_capnp(&id)) - .collect(), + from_capnp_list!(res.get()?, get_finished_tasks, TaskId), + from_capnp_list!(res.get()?, get_finished_objects, DataObjectId), )) } diff --git a/src/client/rpc.rs b/src/client/rpc.rs index 0784f11..082fc12 100644 --- a/src/client/rpc.rs +++ b/src/client/rpc.rs @@ -4,7 +4,7 @@ use client::task::TaskInput; use common::convert::ToCapnp; use common::id::DataObjectId; -macro_rules! capnplist { +macro_rules! to_capnp_list { ($builder:expr, $items:expr, $name:ident) => { { let mut builder = $builder.$name($items.len() as u32); @@ -14,6 +14,17 @@ macro_rules! capnplist { } } } +macro_rules! from_capnp_list { + ($builder:expr, $items:ident, $obj:ident) => { + { + $builder + .$items()? + .iter() + .map(|item| $obj::from_capnp(&item)) + .collect() + } + } +} impl<'a> ToCapnp<'a> for TaskInput { type Builder = ::client_capnp::task::in_data_object::Builder<'a>; @@ -36,8 +47,8 @@ impl<'a> ToCapnp<'a> for Task { self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); builder.set_task_type(&self.command); - capnplist!(builder.reborrow(), self.inputs, init_inputs); - capnplist!( + to_capnp_list!(builder.reborrow(), self.inputs, init_inputs); + to_capnp_list!( builder.reborrow(), self.outputs .iter() From ffa94b1d135e32e15a435c2b729fc809e414e9f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 4 May 2018 13:22:31 +0200 Subject: [PATCH 16/18] [api] use Rc for Tasks in session --- src/client/client.rs | 10 +++---- src/client/communicator.rs | 53 ++++++++++++++++++++++++-------------- src/client/session.rs | 37 ++++++++++---------------- 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/client/client.rs b/src/client/client.rs index 976adc8..6d46e3c 100644 --- a/src/client/client.rs +++ b/src/client/client.rs @@ -2,27 +2,27 @@ use std::error::Error; use std::net::SocketAddr; use CLIENT_PROTOCOL_VERSION; -use common::wrapped::WrappedRcRefCell; use super::session::Session; use super::communicator::Communicator; +use std::rc::Rc; pub struct Client { - comm: WrappedRcRefCell, + comm: Rc, } impl Client { pub fn new(scheduler: SocketAddr) -> Result> { - let comm = WrappedRcRefCell::wrap(Communicator::new(scheduler, CLIENT_PROTOCOL_VERSION)?); + let comm = Rc::new(Communicator::new(scheduler, CLIENT_PROTOCOL_VERSION)?); Ok(Client { comm }) } pub fn new_session(&self) -> Result> { - let session_id = self.comm.get_mut().new_session()?; + let session_id = self.comm.new_session()?; Ok(Session::new(session_id, self.comm.clone())) } pub fn terminate_server(&self) -> Result<(), Box> { - self.comm.get_mut().terminate_server() + self.comm.terminate_server() } } diff --git a/src/client/communicator.rs b/src/client/communicator.rs index 6024d20..c1cd29a 100644 --- a/src/client/communicator.rs +++ b/src/client/communicator.rs @@ -5,15 +5,16 @@ use std::error::Error; use common::rpc::new_rpc_system; use capnp_rpc::rpc_twoparty_capnp; use futures::Future; -use std::cell::Ref; use super::task::Task; use common::id::{DataObjectId, TaskId}; use common::convert::{FromCapnp, ToCapnp}; use client::dataobject::DataObject; +use std::cell::RefCell; +use std::cell::RefMut; pub struct Communicator { - core: Core, + core: RefCell, service: ::client_capnp::client_service::Client, } @@ -36,19 +37,22 @@ impl Communicator { let service = core.run(request.send().promise)?.get()?.get_service()?; - Ok(Self { core, service }) + Ok(Self { + core: RefCell::new(core), + service, + }) } - pub fn new_session(&mut self) -> Result> { - let id: i32 = self.core + pub fn new_session(&self) -> Result> { + let id: i32 = self.comm() .run(self.service.new_session_request().send().promise)? .get()? .get_session_id(); Ok(id) } - pub fn close_session(&mut self, id: i32) -> Result<(), Box> { - self.core.run({ + pub fn close_session(&self, id: i32) -> Result<(), Box> { + self.comm().run({ let mut req = self.service.close_session_request(); req.get().set_session_id(id); req.send().promise @@ -57,13 +61,18 @@ impl Communicator { Ok(()) } - pub fn submit(&mut self, tasks: &[Ref], data_objects: &[D]) -> Result<(), Box> + pub fn submit(&self, tasks: &[T], data_objects: &[D]) -> Result<(), Box> where + T: AsRef, D: AsRef, { let mut req = self.service.submit_request(); - to_capnp_list!(req.get(), tasks, init_tasks); + to_capnp_list!( + req.get(), + tasks.iter().map(|t| t.as_ref()).collect::>(), + init_tasks + ); to_capnp_list!( req.get(), data_objects @@ -72,34 +81,34 @@ impl Communicator { .collect::>(), init_objects ); - self.core.run(req.send().promise)?; + self.comm().run(req.send().promise)?; Ok(()) } - pub fn unkeep(&mut self, objects: &[DataObjectId]) -> Result<(), Box> { + pub fn unkeep(&self, objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.unkeep_request(); to_capnp_list!(req.get(), objects, init_object_ids); - self.core.run(req.send().promise)?; + self.comm().run(req.send().promise)?; Ok(()) } - pub fn wait(&mut self, tasks: &[TaskId], objects: &[DataObjectId]) -> Result<(), Box> { + pub fn wait(&self, tasks: &[TaskId], objects: &[DataObjectId]) -> Result<(), Box> { let mut req = self.service.wait_request(); to_capnp_list!(req.get(), tasks, init_task_ids); to_capnp_list!(req.get(), objects, init_object_ids); - self.core.run(req.send().promise)?; + self.comm().run(req.send().promise)?; Ok(()) } pub fn wait_some( - &mut self, + &self, tasks: &[TaskId], objects: &[DataObjectId], ) -> Result<(Vec, Vec), Box> { let mut req = self.service.wait_some_request(); to_capnp_list!(req.get(), tasks, init_task_ids); to_capnp_list!(req.get(), objects, init_object_ids); - let res = self.core.run(req.send().promise)?; + let res = self.comm().run(req.send().promise)?; Ok(( from_capnp_list!(res.get()?, get_finished_tasks, TaskId), @@ -107,12 +116,12 @@ impl Communicator { )) } - pub fn fetch(&mut self, object_id: DataObjectId) -> Result, Box> { + pub fn fetch(&self, object_id: DataObjectId) -> Result, Box> { let mut req = self.service.fetch_request(); object_id.to_capnp(&mut req.get().get_id().unwrap()); req.get().set_size(1024); - let response = self.core.run(req.send().promise)?; + let response = self.comm().run(req.send().promise)?; let reader = response.get()?; match reader.get_status().which()? { @@ -124,9 +133,13 @@ impl Communicator { } } - pub fn terminate_server(&mut self) -> Result<(), Box> { - self.core + pub fn terminate_server(&self) -> Result<(), Box> { + self.comm() .run(self.service.terminate_server_request().send().promise)?; Ok(()) } + + fn comm(&self) -> RefMut { + self.core.borrow_mut() + } } diff --git a/src/client/session.rs b/src/client/session.rs index 76f4503..4a16f0d 100644 --- a/src/client/session.rs +++ b/src/client/session.rs @@ -1,5 +1,3 @@ -use common::wrapped::WrappedRcRefCell; - use super::communicator::Communicator; use client::dataobject::DataObject; use client::task::Task; @@ -11,22 +9,22 @@ use common::id::TaskId; use common::id::DataObjectId; use common::id::SId; use common::DataType; -use std::cell::{Cell, Ref}; +use std::cell::Cell; use std::rc::Rc; pub type DataObjectPtr = Rc; -pub type TaskPtr = WrappedRcRefCell; +pub type TaskPtr = Rc; pub struct Session { pub id: i32, - comm: WrappedRcRefCell, + comm: Rc, tasks: Vec, data_objects: Vec, id_counter: i32, } impl Session { - pub fn new(id: i32, comm: WrappedRcRefCell) -> Self { + pub fn new(id: i32, comm: Rc) -> Self { debug!("Session {} created", id); Session { @@ -39,13 +37,7 @@ impl Session { } pub fn submit(&mut self) -> Result<(), Box> { - self.comm.get_mut().submit( - &self.tasks - .iter() - .map(|t| t.get()) - .collect::>>(), - &self.data_objects, - )?; + self.comm.submit(&self.tasks, &self.data_objects)?; self.tasks.clear(); self.data_objects.clear(); @@ -54,13 +46,12 @@ impl Session { pub fn unkeep(&mut self, objects: &[DataObjectPtr]) -> Result<(), Box> { self.comm - .get_mut() .unkeep(&objects.iter().map(|o| o.id).collect::>()) } pub fn wait(&mut self, tasks: &[TaskPtr], objects: &[DataObjectPtr]) -> Result<(), Box> { - self.comm.get_mut().wait( - &tasks.iter().map(|t| t.get().id).collect::>(), + self.comm.wait( + &tasks.iter().map(|t| t.id).collect::>(), &objects.iter().map(|o| o.id).collect::>(), ) } @@ -69,12 +60,12 @@ impl Session { tasks: &[TaskPtr], objects: &[DataObjectPtr], ) -> Result<(Vec, Vec), Box> { - let task_map: HashMap = tasks.iter().map(|t| (t.get().id, t)).collect(); + let task_map: HashMap = tasks.iter().map(|t| (t.id, t)).collect(); let object_map: HashMap = objects.iter().map(|o| (o.id, o)).collect(); - let (task_ids, object_ids) = self.comm.get_mut().wait_some( - &tasks.iter().map(|t| t.get().id).collect::>(), + let (task_ids, object_ids) = self.comm.wait_some( + &tasks.iter().map(|t| t.id).collect::>(), &objects.iter().map(|o| o.id).collect::>(), )?; @@ -90,14 +81,14 @@ impl Session { )) } pub fn wait_all(&mut self) -> Result<(), Box> { - self.comm.get_mut().wait( + self.comm.wait( &vec![TaskId::new(self.id, ::common_capnp::ALL_TASKS_ID)], &vec![], ) } pub fn fetch(&mut self, object: &DataObject) -> Result, Box> { - self.comm.get_mut().fetch(object.id) + self.comm.fetch(object.id) } pub fn blob(&mut self, data: Vec) -> DataObjectPtr { @@ -154,7 +145,7 @@ impl Session { attributes, }; - let rc = WrappedRcRefCell::wrap(task); + let rc = Rc::new(task); self.tasks.push(rc.clone()); rc @@ -163,7 +154,7 @@ impl Session { impl Drop for Session { fn drop(&mut self) { - self.comm.get_mut().close_session(self.id).unwrap(); + self.comm.close_session(self.id).unwrap(); debug!("Session {} destroyed", self.id); } } From abdfda210a8dc66736df6e2b3bc11a8edb11fb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 7 Jul 2018 15:11:01 +0200 Subject: [PATCH 17/18] [api] use rain_core in rust client --- Cargo.lock | 22 +++ Cargo.toml | 3 + rain_client/Cargo.toml | 31 ++++ {src => rain_client/src}/client/client.rs | 5 +- .../src}/client/communicator.rs | 35 +++-- {src => rain_client/src}/client/dataobject.rs | 12 +- rain_client/src/client/localcluster.rs | 49 +++++++ {src => rain_client/src}/client/mod.rs | 2 + rain_client/src/client/rpc.rs | 46 ++++++ {src => rain_client/src}/client/session.rs | 79 +++++----- rain_client/src/client/task.rs | 18 +++ {src => rain_client/src}/client/tasks.rs | 24 +-- rain_client/src/lib.rs | 12 ++ rain_client_test/Cargo.toml | 10 ++ rain_client_test/src/main.rs | 27 ++++ rain_core/Cargo.toml | 1 + rain_core/src/comm/executor.rs | 2 +- rain_core/src/comm/mod.rs | 2 + rain_core/src/comm/rpc.rs | 19 +++ rain_core/src/lib.rs | 3 +- rain_server/src/common/connection.rs | 2 +- rain_server/src/governor/graph/dataobj.rs | 2 +- rain_server/src/governor/rpc/control.rs | 2 +- rain_server/src/governor/rpc/executor.rs | 2 +- rain_server/src/governor/rpc/fetch.rs | 4 +- rain_server/src/governor/state.rs | 4 +- rain_server/src/main.rs | 11 +- rain_server/src/server/http.rs | 4 +- .../src/server/logging/sqlite_logger.rs | 2 +- rain_server/src/server/rpc/client.rs | 1 + rain_server/src/start/common.rs | 33 ----- rain_server/src/start/mod.rs | 1 - rain_server/src/start/starter.rs | 6 - rain_task/src/macros.rs | 3 +- rain_task/src/tests.rs | 137 ++++++++++-------- src/client/rpc.rs | 82 ----------- src/client/task.rs | 27 ---- src/start/localcluster.rs | 55 ------- 38 files changed, 423 insertions(+), 357 deletions(-) create mode 100644 rain_client/Cargo.toml rename {src => rain_client/src}/client/client.rs (94%) rename {src => rain_client/src}/client/communicator.rs (83%) rename {src => rain_client/src}/client/dataobject.rs (50%) create mode 100644 rain_client/src/client/localcluster.rs rename {src => rain_client/src}/client/mod.rs (73%) create mode 100644 rain_client/src/client/rpc.rs rename {src => rain_client/src}/client/session.rs (68%) create mode 100644 rain_client/src/client/task.rs rename {src => rain_client/src}/client/tasks.rs (64%) create mode 100644 rain_client/src/lib.rs create mode 100644 rain_client_test/Cargo.toml create mode 100644 rain_client_test/src/main.rs create mode 100644 rain_core/src/comm/rpc.rs delete mode 100644 src/client/rpc.rs delete mode 100644 src/client/task.rs delete mode 100644 src/start/localcluster.rs diff --git a/Cargo.lock b/Cargo.lock index 1c1fd29..28716e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -555,6 +555,27 @@ dependencies = [ "proc-macro2 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rain_client" +version = "0.3.0" +dependencies = [ + "capnp-rpc 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rain_core 0.3.0", + "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rain_client_test" +version = "0.3.0" +dependencies = [ + "rain_client 0.3.0", +] + [[package]] name = "rain_core" version = "0.3.0" @@ -577,6 +598,7 @@ dependencies = [ "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 7683228..047ef1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,8 @@ version = "0.3.0" [workspace] members = [ + "rain_client", + "rain_client_test", "rain_core", "rain_server", "rain_task", @@ -10,5 +12,6 @@ members = [ ] [patch.crates-io] +rain_client = { path = "rain_client" } rain_core = { path = "rain_core" } rain_task = { path = "rain_task" } diff --git a/rain_client/Cargo.toml b/rain_client/Cargo.toml new file mode 100644 index 0000000..cc49275 --- /dev/null +++ b/rain_client/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "rain_client" +version = "0.3.0" + +description = "Distributed computational framework for large-scale task-based pipelines. Client library in Rust." +# documentation = "https://docs.rs/rain_task/" # default docs.rs +homepage = "https://github.com/substantic/rain" +repository = "https://github.com/substantic/rain/" +readme = "README.md" +authors = [ + "Stanislav Bohm ", + "Tomas Gavenciak ", + "Vojtech Cima ", + ] +license = "MIT" + +exclude = ["testing/**/*"] + +[badges] +travis-ci = { repository = "substantic/rain", branch = "master" } +maintenance = { status = "actively-developed" } + +[dependencies] +capnp-rpc = "0.8" +error-chain = "0.11" +futures = "0.1" +log = "0.4" +rain_core = "0.3.0" +tokio-core = "0.1" +serde = "1" +serde_json = "1" diff --git a/src/client/client.rs b/rain_client/src/client/client.rs similarity index 94% rename from src/client/client.rs rename to rain_client/src/client/client.rs index 6d46e3c..267af9c 100644 --- a/src/client/client.rs +++ b/rain_client/src/client/client.rs @@ -1,9 +1,9 @@ use std::error::Error; use std::net::SocketAddr; -use CLIENT_PROTOCOL_VERSION; -use super::session::Session; use super::communicator::Communicator; +use super::session::Session; +use rain_core::CLIENT_PROTOCOL_VERSION; use std::rc::Rc; pub struct Client { @@ -26,3 +26,4 @@ impl Client { self.comm.terminate_server() } } + diff --git a/src/client/communicator.rs b/rain_client/src/client/communicator.rs similarity index 83% rename from src/client/communicator.rs rename to rain_client/src/client/communicator.rs index c1cd29a..ba3ba95 100644 --- a/src/client/communicator.rs +++ b/rain_client/src/client/communicator.rs @@ -1,21 +1,21 @@ -use tokio_core::reactor::Core; -use std::net::SocketAddr; -use tokio_core::net::TcpStream; -use std::error::Error; -use common::rpc::new_rpc_system; use capnp_rpc::rpc_twoparty_capnp; use futures::Future; +use rain_core::comm::new_rpc_system; +use std::error::Error; +use std::net::SocketAddr; +use tokio_core::net::TcpStream; +use tokio_core::reactor::Core; use super::task::Task; -use common::id::{DataObjectId, TaskId}; -use common::convert::{FromCapnp, ToCapnp}; use client::dataobject::DataObject; -use std::cell::RefCell; -use std::cell::RefMut; +use rain_core::types::{DataObjectId, TaskId}; +use rain_core::utils::{FromCapnp, ToCapnp}; +use rain_core::{client_capnp, common_capnp, server_capnp}; +use std::cell::{RefCell, RefMut}; pub struct Communicator { core: RefCell, - service: ::client_capnp::client_service::Client, + service: client_capnp::client_service::Client, } impl Communicator { @@ -28,7 +28,7 @@ impl Communicator { debug!("Connection to server {} established", scheduler); let mut rpc = Box::new(new_rpc_system(stream, None)); - let bootstrap: ::server_capnp::server_bootstrap::Client = + let bootstrap: server_capnp::server_bootstrap::Client = rpc.bootstrap(rpc_twoparty_capnp::Side::Server); handle.spawn(rpc.map_err(|err| panic!("RPC error: {}", err))); @@ -116,19 +116,28 @@ impl Communicator { )) } - pub fn fetch(&self, object_id: DataObjectId) -> Result, Box> { + pub fn fetch(&self, object_id: &DataObjectId) -> Result, Box> { let mut req = self.service.fetch_request(); object_id.to_capnp(&mut req.get().get_id().unwrap()); req.get().set_size(1024); let response = self.comm().run(req.send().promise)?; + // TODO: handle error states let reader = response.get()?; match reader.get_status().which()? { - ::common_capnp::fetch_result::status::Ok(()) => { + common_capnp::fetch_result::status::Ok(()) => { let data = reader.get_data()?; Ok(Vec::from(data)) } + common_capnp::fetch_result::status::Removed(()) => { + print!("Removed"); + Ok(vec![]) + } + common_capnp::fetch_result::status::Error(err) => { + print!("Error: {:?}", err.unwrap().get_message()); + Ok(vec![]) + } _ => bail!("Non-ok status"), } } diff --git a/src/client/dataobject.rs b/rain_client/src/client/dataobject.rs similarity index 50% rename from src/client/dataobject.rs rename to rain_client/src/client/dataobject.rs index 0fdc643..de19c64 100644 --- a/src/client/dataobject.rs +++ b/rain_client/src/client/dataobject.rs @@ -1,19 +1,17 @@ -use common::Attributes; -use common::id::DataObjectId; -use common::DataType; +use rain_core::types::{DataObjectId, ObjectSpec}; use std::cell::Cell; pub struct DataObject { - pub id: DataObjectId, - pub label: String, pub keep: Cell, pub data: Option>, - pub attributes: Attributes, - pub data_type: DataType, + pub spec: ObjectSpec, } impl DataObject { pub fn keep(&self) { self.keep.set(true); } + pub fn id(&self) -> DataObjectId { + self.spec.id + } } diff --git a/rain_client/src/client/localcluster.rs b/rain_client/src/client/localcluster.rs new file mode 100644 index 0000000..01c92c4 --- /dev/null +++ b/rain_client/src/client/localcluster.rs @@ -0,0 +1,49 @@ +use std::error::Error; +use std::net::SocketAddr; +use super::client::Client; +use std::process::{Child, Command}; +use std::path::PathBuf; +use std::{thread, time}; +use std::process::Stdio; + +pub struct LocalCluster { + listen_addr: SocketAddr, + binary: PathBuf +} + +impl LocalCluster { + pub fn new(binary: &str) -> Result> { + let mut cluster = LocalCluster { + binary: PathBuf::from(binary), + listen_addr: SocketAddr::new("127.0.0.1".parse()?, 7210) + }; + cluster.start()?; + + Ok(cluster) + } + + fn start(&mut self) -> Result<(), Box> { + Command::new(&self.binary) + .arg("start") + .arg("--listen") + .arg(self.listen_addr.to_string()) + .arg("--simple") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()? + .wait()?; + + Ok(()) + } + + pub fn create_client(&self) -> Result> { + Client::new(self.listen_addr) + } +} + +impl Drop for LocalCluster { + fn drop(&mut self) { + Client::new(self.listen_addr).unwrap().terminate_server(); + } +} diff --git a/src/client/mod.rs b/rain_client/src/client/mod.rs similarity index 73% rename from src/client/mod.rs rename to rain_client/src/client/mod.rs index 975fed6..d40fe1d 100644 --- a/src/client/mod.rs +++ b/rain_client/src/client/mod.rs @@ -1,6 +1,7 @@ pub mod client; pub mod session; pub mod tasks; +pub mod localcluster; #[macro_use] mod rpc; @@ -10,3 +11,4 @@ mod task; pub use self::client::Client; pub use self::session::Session; +pub use self::localcluster::LocalCluster; diff --git a/rain_client/src/client/rpc.rs b/rain_client/src/client/rpc.rs new file mode 100644 index 0000000..9cf15f8 --- /dev/null +++ b/rain_client/src/client/rpc.rs @@ -0,0 +1,46 @@ +use super::task::Task; +use client::dataobject::DataObject; +use rain_core::client_capnp; +use rain_core::utils::ToCapnp; +use serde_json; + +macro_rules! to_capnp_list { + ($builder:expr, $items:expr, $name:ident) => {{ + let mut builder = $builder.$name($items.len() as u32); + for (i, obj) in $items.iter().enumerate() { + obj.to_capnp(&mut builder.reborrow().get(i as u32)); + } + }}; +} +macro_rules! from_capnp_list { + ($builder:expr, $items:ident, $obj:ident) => {{ + $builder + .$items()? + .iter() + .map(|item| $obj::from_capnp(&item)) + .collect() + }}; +} + +impl<'a> ToCapnp<'a> for Task { + type Builder = client_capnp::task::Builder<'a>; + + fn to_capnp(&self, builder: &mut Self::Builder) { + builder.set_spec(&serde_json::to_string(&self.spec).unwrap()); + } +} +impl<'a> ToCapnp<'a> for DataObject { + type Builder = client_capnp::data_object::Builder<'a>; + + fn to_capnp(&self, builder: &mut Self::Builder) { + builder.set_spec(&serde_json::to_string(&self.spec).unwrap()); + builder.set_keep(self.keep.get()); + + if let &Some(ref data) = &self.data { + builder.set_data(&data); + builder.set_has_data(true); + } else { + builder.set_has_data(false); + } + } +} diff --git a/src/client/session.rs b/rain_client/src/client/session.rs similarity index 68% rename from src/client/session.rs rename to rain_client/src/client/session.rs index 4a16f0d..9097f07 100644 --- a/src/client/session.rs +++ b/rain_client/src/client/session.rs @@ -1,15 +1,13 @@ use super::communicator::Communicator; use client::dataobject::DataObject; use client::task::Task; +use rain_core::common_capnp; +use rain_core::types::{DataObjectId, DataType, ObjectSpec, Resources, SId, TaskId, TaskSpec, + TaskSpecInput, UserAttrs}; +use serde_json; +use std::cell::Cell; use std::collections::HashMap; use std::error::Error; -use client::task::TaskInput; -use common::Attributes; -use common::id::TaskId; -use common::id::DataObjectId; -use common::id::SId; -use common::DataType; -use std::cell::Cell; use std::rc::Rc; pub type DataObjectPtr = Rc; @@ -45,14 +43,19 @@ impl Session { } pub fn unkeep(&mut self, objects: &[DataObjectPtr]) -> Result<(), Box> { - self.comm - .unkeep(&objects.iter().map(|o| o.id).collect::>()) + self.comm.unkeep(&objects + .iter() + .map(|o| o.id()) + .collect::>()) } pub fn wait(&mut self, tasks: &[TaskPtr], objects: &[DataObjectPtr]) -> Result<(), Box> { self.comm.wait( - &tasks.iter().map(|t| t.id).collect::>(), - &objects.iter().map(|o| o.id).collect::>(), + &tasks.iter().map(|t| t.id()).collect::>(), + &objects + .iter() + .map(|o| o.id()) + .collect::>(), ) } pub fn wait_some( @@ -60,13 +63,16 @@ impl Session { tasks: &[TaskPtr], objects: &[DataObjectPtr], ) -> Result<(Vec, Vec), Box> { - let task_map: HashMap = tasks.iter().map(|t| (t.id, t)).collect(); + let task_map: HashMap = tasks.iter().map(|t| (t.id(), t)).collect(); let object_map: HashMap = - objects.iter().map(|o| (o.id, o)).collect(); + objects.iter().map(|o| (o.id(), o)).collect(); let (task_ids, object_ids) = self.comm.wait_some( - &tasks.iter().map(|t| t.id).collect::>(), - &objects.iter().map(|o| o.id).collect::>(), + &tasks.iter().map(|t| t.id()).collect::>(), + &objects + .iter() + .map(|o| o.id()) + .collect::>(), )?; Ok(( @@ -82,13 +88,13 @@ impl Session { } pub fn wait_all(&mut self) -> Result<(), Box> { self.comm.wait( - &vec![TaskId::new(self.id, ::common_capnp::ALL_TASKS_ID)], + &vec![TaskId::new(self.id, common_capnp::ALL_TASKS_ID)], &vec![], ) } - pub fn fetch(&mut self, object: &DataObject) -> Result, Box> { - self.comm.fetch(object.id) + pub fn fetch(&mut self, o: &DataObjectPtr) -> Result, Box> { + self.comm.fetch(&o.id()) } pub fn blob(&mut self, data: Vec) -> DataObjectPtr { @@ -102,13 +108,17 @@ impl Session { DataObjectId::new(self.id, id) } pub(crate) fn create_object(&mut self, label: String, data: Option>) -> DataObjectPtr { - let object = DataObject { + let spec = ObjectSpec { id: self.create_object_id(), - keep: Cell::new(false), label, - data, - attributes: Attributes::new(), data_type: DataType::Blob, + content_type: "".to_owned(), + user: UserAttrs::new(), + }; + let object = DataObject { + keep: Cell::new(false), + data, + spec, }; let rc = Rc::new(object); self.data_objects.push(rc.clone()); @@ -124,27 +134,24 @@ impl Session { } pub fn create_task( &mut self, - command: String, - inputs: Vec, + task_type: String, + inputs: Vec, outputs: Vec, config: HashMap, - cpus: i32, + cpus: u32, ) -> TaskPtr { - let mut attributes = Attributes::new(); - attributes.set("config", config).unwrap(); - - let mut resources: HashMap = HashMap::new(); - resources.insert("cpus".to_owned(), cpus); - attributes.set("resources", resources).unwrap(); - - let task = Task { + let spec = TaskSpec { id: self.create_task_id(), - command, inputs, - outputs, - attributes, + task_type, + outputs: outputs.iter().map(|o| o.id()).collect(), + config: Some(serde_json::to_value(&config).unwrap()), + resources: Resources { cpus }, + user: UserAttrs::new(), }; + let task = Task { spec, outputs }; + let rc = Rc::new(task); self.tasks.push(rc.clone()); diff --git a/rain_client/src/client/task.rs b/rain_client/src/client/task.rs new file mode 100644 index 0000000..e31cc3d --- /dev/null +++ b/rain_client/src/client/task.rs @@ -0,0 +1,18 @@ +use super::session::DataObjectPtr; +use rain_core::types::{TaskId, TaskSpec}; + +pub struct Task { + pub spec: TaskSpec, + pub outputs: Vec, +} + +impl Task { + pub fn output(&self) -> DataObjectPtr { + assert_eq!(self.outputs.len(), 1, "Task has multiple outputs"); + + self.outputs[0].clone() + } + pub fn id(&self) -> TaskId { + self.spec.id + } +} diff --git a/src/client/tasks.rs b/rain_client/src/client/tasks.rs similarity index 64% rename from src/client/tasks.rs rename to rain_client/src/client/tasks.rs index 1926857..1cfe687 100644 --- a/src/client/tasks.rs +++ b/rain_client/src/client/tasks.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; -use super::task::TaskInput; use super::session::{DataObjectPtr, Session, TaskPtr}; +use rain_core::types::TaskSpecInput; pub trait CommonTasks { fn concat(&mut self, objects: &[DataObjectPtr]) -> TaskPtr; @@ -9,19 +9,23 @@ pub trait CommonTasks { fn export(&mut self, object: DataObjectPtr, filename: String) -> TaskPtr; } +fn builtin(action: &str) -> String { + return format!("buildin/{}", action); +} + impl CommonTasks for Session { fn concat(&mut self, objects: &[DataObjectPtr]) -> TaskPtr { let inputs = objects .iter() - .map(|o| TaskInput { - label: None, - data_object: o.clone(), + .map(|o| TaskSpecInput { + label: "".to_owned(), + id: o.id(), }) .collect(); let outputs = vec![self.create_object("".to_owned(), None)]; - self.create_task("!concat".to_owned(), inputs, outputs, HashMap::new(), 1) + self.create_task(builtin("concat"), inputs, outputs, HashMap::new(), 1) } fn open(&mut self, filename: String) -> TaskPtr { let mut config = HashMap::new(); @@ -29,17 +33,17 @@ impl CommonTasks for Session { let outputs = vec![self.create_object("".to_owned(), None)]; - self.create_task("!open".to_owned(), vec![], outputs, config, 1) + self.create_task(builtin("open"), vec![], outputs, config, 1) } fn export(&mut self, object: DataObjectPtr, filename: String) -> TaskPtr { let mut config = HashMap::new(); config.insert("path".to_owned(), filename); - let input = TaskInput { - label: None, - data_object: object.clone(), + let input = TaskSpecInput { + label: "".to_owned(), + id: object.id(), }; - self.create_task("!export".to_owned(), vec![input], vec![], config, 1) + self.create_task(builtin("export"), vec![input], vec![], config, 1) } } diff --git a/rain_client/src/lib.rs b/rain_client/src/lib.rs new file mode 100644 index 0000000..cb390ec --- /dev/null +++ b/rain_client/src/lib.rs @@ -0,0 +1,12 @@ +extern crate capnp_rpc; +#[macro_use] +extern crate error_chain; +extern crate futures; +#[macro_use] +extern crate log; +extern crate rain_core; +extern crate serde_json; +extern crate tokio_core; + +pub mod client; +pub use client::*; diff --git a/rain_client_test/Cargo.toml b/rain_client_test/Cargo.toml new file mode 100644 index 0000000..57ebf91 --- /dev/null +++ b/rain_client_test/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rain_client_test" +version = "0.3.0" +authors = ["Jakub Beranek "] +description = "A testing Rain client in Rust." +license = "MIT" +publish = false + +[dependencies] +rain_client = "0.3.0" diff --git a/rain_client_test/src/main.rs b/rain_client_test/src/main.rs new file mode 100644 index 0000000..d12eef0 --- /dev/null +++ b/rain_client_test/src/main.rs @@ -0,0 +1,27 @@ +extern crate rain_client; + +use std::error::Error; +use rain_client::tasks::CommonTasks; +use rain_client::client::LocalCluster; + +fn test() -> Result<(), Box> { + let cluster = LocalCluster::new("/home/kobzol/projects/it4i/rain/target/debug/rain")?; + + let client = cluster.create_client()?; + let mut s = client.new_session()?; + + let a = s.open("/tmp/asd.txt".to_owned()); + let b = s.open("/tmp/asd2.txt".to_owned()); + + let c = s.concat(&[a.output(), b.output()]); + c.output().keep(); + s.submit()?; + let res = s.fetch(&c.output())?; + println!("{}", String::from_utf8(res)?); + + Ok(()) +} + +fn main() { + test().unwrap(); +} diff --git a/rain_core/Cargo.toml b/rain_core/Cargo.toml index 01cdae3..db4daa9 100644 --- a/rain_core/Cargo.toml +++ b/rain_core/Cargo.toml @@ -38,6 +38,7 @@ serde_cbor = "0.8" serde_derive = "1.0" serde_json = "1.0" tokio-core="0.1" +tokio-io="0.1" tokio-timer = "0.2" [build-dependencies] diff --git a/rain_core/src/comm/executor.rs b/rain_core/src/comm/executor.rs index 6dbf044..f64f7ee 100644 --- a/rain_core/src/comm/executor.rs +++ b/rain_core/src/comm/executor.rs @@ -143,7 +143,7 @@ pub struct DropCachedMsg { #[cfg(test)] mod tests { use super::*; - use serde::{de::DeserializeOwned, Serialize}; + use serde::{Serialize, de::DeserializeOwned}; use serde_cbor; use serde_json; use std::fmt::Debug; diff --git a/rain_core/src/comm/mod.rs b/rain_core/src/comm/mod.rs index cac5179..c087a25 100644 --- a/rain_core/src/comm/mod.rs +++ b/rain_core/src/comm/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod executor; +pub mod rpc; pub use self::executor::{CallMsg, DataLocation, DropCachedMsg, ExecutorToGovernorMessage, GovernorToExecutorMessage, LocalObjectIn, LocalObjectOut, RegisterMsg, ResultMsg}; +pub use self::rpc::new_rpc_system; diff --git a/rain_core/src/comm/rpc.rs b/rain_core/src/comm/rpc.rs new file mode 100644 index 0000000..fed2a2d --- /dev/null +++ b/rain_core/src/comm/rpc.rs @@ -0,0 +1,19 @@ +use capnp_rpc::{rpc_twoparty_capnp, twoparty, RpcSystem}; +use tokio_io::{AsyncRead, AsyncWrite}; + +pub fn new_rpc_system( + stream: Stream, + bootstrap: Option<::capnp::capability::Client>, +) -> RpcSystem +where + Stream: AsyncRead + AsyncWrite + 'static, +{ + let (reader, writer) = stream.split(); + let network = Box::new(twoparty::VatNetwork::new( + reader, + writer, + rpc_twoparty_capnp::Side::Client, + Default::default(), + )); + RpcSystem::new(network, bootstrap) +} diff --git a/rain_core/src/lib.rs b/rain_core/src/lib.rs index ff60b45..b212178 100644 --- a/rain_core/src/lib.rs +++ b/rain_core/src/lib.rs @@ -3,7 +3,7 @@ //! This documentation is minimalistic but still hopefully useful. //! As an user, you may be interested in the //! [rain_task lirary documentation](https://docs.rs/rain_task/). -//! +//! //! See `README.md` and the [project page](https://github.com/substantic/rain/) //! for general information. @@ -30,6 +30,7 @@ extern crate serde_cbor; extern crate serde_derive; extern crate serde_json; extern crate tokio_core; +extern crate tokio_io; extern crate tokio_timer; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/rain_server/src/common/connection.rs b/rain_server/src/common/connection.rs index ab7ac4f..0cf3f8c 100644 --- a/rain_server/src/common/connection.rs +++ b/rain_server/src/common/connection.rs @@ -1,8 +1,8 @@ use bytes::BytesMut; use futures::{Future, Stream}; -use tokio_io::codec::length_delimited::{Builder, Framed}; use tokio_io::AsyncRead; use tokio_io::AsyncWrite; +use tokio_io::codec::length_delimited::{Builder, Framed}; use rain_core::errors::{Error, Result}; diff --git a/rain_server/src/governor/graph/dataobj.rs b/rain_server/src/governor/graph/dataobj.rs index d6081b5..64d5325 100644 --- a/rain_server/src/governor/graph/dataobj.rs +++ b/rain_server/src/governor/graph/dataobj.rs @@ -5,9 +5,9 @@ use std::path::Path; use std::sync::Arc; use super::{Graph, TaskRef}; +use governor::WorkDir; use governor::data::Data; use governor::graph::ExecutorRef; -use governor::WorkDir; use wrapped::WrappedRcRefCell; #[derive(Debug)] diff --git a/rain_server/src/governor/rpc/control.rs b/rain_server/src/governor/rpc/control.rs index 90ee9eb..ba3ff12 100644 --- a/rain_server/src/governor/rpc/control.rs +++ b/rain_server/src/governor/rpc/control.rs @@ -3,8 +3,8 @@ use futures::future::Future; use rain_core::{errors::*, types::*, utils::*}; use std::sync::Arc; -use governor::graph::DataObjectState; use governor::StateRef; +use governor::graph::DataObjectState; use rain_core::governor_capnp::governor_control; pub struct GovernorControlImpl { diff --git a/rain_server/src/governor/rpc/executor.rs b/rain_server/src/governor/rpc/executor.rs index ed5f6c3..20fff0a 100644 --- a/rain_server/src/governor/rpc/executor.rs +++ b/rain_server/src/governor/rpc/executor.rs @@ -3,8 +3,8 @@ use rain_core::{comm::*, errors::*, types::*}; use std::path::Path; use std::sync::Arc; -use governor::data::{Data, Storage}; use governor::State; +use governor::data::{Data, Storage}; static PROTOCOL_VERSION: &'static str = "cbor-1"; diff --git a/rain_server/src/governor/rpc/fetch.rs b/rain_server/src/governor/rpc/fetch.rs index ca0af8a..ecc70b4 100644 --- a/rain_server/src/governor/rpc/fetch.rs +++ b/rain_server/src/governor/rpc/fetch.rs @@ -1,12 +1,12 @@ -use futures::future::Either; use futures::IntoFuture; +use futures::future::Either; use futures::{future, Future}; use rain_core::{errors::*, types::*, utils::*}; use std::rc::Rc; +use governor::StateRef; use governor::data::{Data, DataBuilder}; use governor::graph::DataObjectRef; -use governor::StateRef; pub struct FetchContext { pub state_ref: StateRef, diff --git a/rain_server/src/governor/state.rs b/rain_server/src/governor/state.rs index fe99231..998144a 100644 --- a/rain_server/src/governor/state.rs +++ b/rain_server/src/governor/state.rs @@ -12,14 +12,14 @@ use std::time::{Duration, Instant}; use common::Monitor; use common::{create_protocol_stream, new_rpc_system, Connection}; -use governor::data::transport::TransportView; use governor::data::Data; +use governor::data::transport::TransportView; use governor::fs::workdir::WorkDir; use governor::graph::executor::get_log_tails; use governor::graph::{executor_command, DataObject, DataObjectRef, DataObjectState, ExecutorRef, Graph, TaskRef, TaskState}; -use governor::rpc::executor::check_registration; use governor::rpc::GovernorControlImpl; +use governor::rpc::executor::check_registration; use governor::tasks::TaskInstance; use wrapped::WrappedRcRefCell; diff --git a/rain_server/src/main.rs b/rain_server/src/main.rs index 7f96d39..94ac0fb 100644 --- a/rain_server/src/main.rs +++ b/rain_server/src/main.rs @@ -12,6 +12,7 @@ extern crate chrono; #[macro_use] extern crate clap; extern crate env_logger; +#[macro_use] extern crate error_chain; extern crate fs_extra; extern crate futures; @@ -60,7 +61,6 @@ use std::process::exit; use rain_core::sys::{create_ready_file, get_hostname}; use rain_core::{errors::*, utils::*}; -use clap::{App, Arg, ArgMatches, SubCommand}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_SERVER_PORT: u16 = 7210; @@ -141,7 +141,6 @@ fn run_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } } -<<<<<<< 7999023457517e7570c2373ab414486b9fe0163c:rain_server/src/main.rs fn default_working_directory() -> PathBuf { let pid = getpid(); let hostname = get_hostname(); @@ -171,8 +170,6 @@ fn ensure_directory(dir: &Path, name: &str) -> Result<()> { Ok(()) } -======= ->>>>>>> [api] add local cluster:src/bin.rs // TODO: Do some serious configuration file and unify configurations // Right now, it is just a quick hack for supporting executors @@ -424,7 +421,7 @@ fn run_starter(_global_args: &ArgMatches, cmd_args: &ArgMatches) { } fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { - let default_address = format!("localhost:{}", DEFAULT_SERVER_PORT); + /*let default_address = format!("localhost:{}", DEFAULT_SERVER_PORT); let mut address = cmd_args .value_of("SERVER_ADDRESS") .unwrap_or(&default_address) @@ -444,7 +441,7 @@ fn stop_server(_global_args: &ArgMatches, cmd_args: &ArgMatches) { exit(1); }); - println!("Server at {} was successfully stopped", address); + println!("Server at {} was successfully stopped", address);*/ } fn init_log() { @@ -611,7 +608,7 @@ fn main() { .takes_value(true))) .subcommand( // ---- STOP ---- SubCommand::with_name("stop") - .about("Stop server and all workers connected to it") + .about("Stop server and all governors connected to it") .arg(Arg::with_name("SERVER_ADDRESS") .help("Address of the server (default = localhost:7210)") .takes_value(true))) diff --git a/rain_server/src/server/http.rs b/rain_server/src/server/http.rs index 5b63579..682f545 100644 --- a/rain_server/src/server/http.rs +++ b/rain_server/src/server/http.rs @@ -145,9 +145,7 @@ impl Service for RequestHandler { &include_bytes!("./../../dashboard/dist/main.css.gz")[..], ) } - _ => static_data_response( - &include_bytes!("./../../dashboard/dist/index.html")[..], - ), + _ => static_data_response(&include_bytes!("./../../dashboard/dist/index.html")[..]), /*path => { warn!("Invalid HTTP request: {}", path); Response::new().with_status(StatusCode::NotFound) diff --git a/rain_server/src/server/logging/sqlite_logger.rs b/rain_server/src/server/logging/sqlite_logger.rs index 2b9cc10..17590bf 100644 --- a/rain_server/src/server/logging/sqlite_logger.rs +++ b/rain_server/src/server/logging/sqlite_logger.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, Utc}; -use futures::sync::{mpsc, oneshot}; use futures::Future; use futures::Stream; +use futures::sync::{mpsc, oneshot}; use rusqlite::Connection; use serde_json; use std::path::PathBuf; diff --git a/rain_server/src/server/rpc/client.rs b/rain_server/src/server/rpc/client.rs index 4f34f10..cb81f41 100644 --- a/rain_server/src/server/rpc/client.rs +++ b/rain_server/src/server/rpc/client.rs @@ -7,6 +7,7 @@ use std::net::SocketAddr; use server::graph::{ClientRef, TaskRef}; use server::graph::{DataObjectRef, DataObjectState}; use server::state::StateRef; +use std::process::exit; pub struct ClientServiceImpl { state: StateRef, diff --git a/rain_server/src/start/common.rs b/rain_server/src/start/common.rs index c96f6c3..10289e0 100644 --- a/rain_server/src/start/common.rs +++ b/rain_server/src/start/common.rs @@ -1,9 +1,4 @@ use std::path::PathBuf; -use nix::unistd::getpid; -use std::path::Path; -use errors::Result; -use std::fs::create_dir_all; -use std::error::Error; pub enum Readiness { /// Ready file is a file that @@ -11,31 +6,3 @@ pub enum Readiness { WaitingForReadyFile(PathBuf), IsReady, } - -pub fn default_working_directory() -> PathBuf { - let pid = getpid(); - let hostname = ::common::sys::get_hostname(); - PathBuf::from("/tmp/rain-work").join(format!("worker-{}-{}", hostname, pid)) -} - -pub fn default_logging_directory(basename: &str) -> PathBuf { - let pid = getpid(); - let hostname = ::common::sys::get_hostname(); - PathBuf::from("/tmp/rain-logs").join(format!("{}-{}-{}", basename, hostname, pid)) -} -pub fn ensure_directory(dir: &Path, name: &str) -> Result<()> { - if !dir.exists() { - debug!("{} not found, creating ... {:?}", name, dir); - if let Err(e) = create_dir_all(dir) { - bail!(format!( - "{} {:?} cannot by created: {}", - name, - dir, - e.description() - )); - } - } else if !dir.is_dir() { - bail!("{} {:?} exists but it is not a directory", name, dir); - } - Ok(()) -} diff --git a/rain_server/src/start/mod.rs b/rain_server/src/start/mod.rs index 0b7d57e..0519e43 100644 --- a/rain_server/src/start/mod.rs +++ b/rain_server/src/start/mod.rs @@ -1,5 +1,4 @@ pub mod common; -pub mod localcluster; pub mod process; pub mod ssh; pub mod starter; diff --git a/rain_server/src/start/starter.rs b/rain_server/src/start/starter.rs index 3024f72..75fe49a 100644 --- a/rain_server/src/start/starter.rs +++ b/rain_server/src/start/starter.rs @@ -11,12 +11,6 @@ use std::process::Command; use start::common::Readiness; use start::process::Process; use start::ssh::RemoteProcess; -use errors::Result; - -use nix::unistd::getpid; -use std::io::BufReader; -use std::io::BufRead; -use std::fs::File; pub struct StarterConfig { /// Number of local governor that will be spawned diff --git a/rain_task/src/macros.rs b/rain_task/src/macros.rs index d75ece6..dd5b261 100644 --- a/rain_task/src/macros.rs +++ b/rain_task/src/macros.rs @@ -88,7 +88,8 @@ macro_rules! register_task_make_call { #[macro_export] macro_rules! register_task { ($executor: expr, $name: expr, [$($params: tt)*], $taskfn: expr) => ({ - $executor.register_task($name, |ctx: &mut Context, ins: &[DataInstance], outs: &mut [Output]| -> TaskResult<()> { + $executor.register_task($name, |ctx: &mut Context, ins: &[DataInstance], + outs: &mut [Output]| -> TaskResult<()> { register_task_make_call!($taskfn, ins, outs, ($($params)*), (ctx)) }) }); diff --git a/rain_task/src/tests.rs b/rain_task/src/tests.rs index dc1462a..d0d175c 100644 --- a/rain_task/src/tests.rs +++ b/rain_task/src/tests.rs @@ -202,7 +202,8 @@ fn register_task() { register_task!(s, "task4", [I I O O], |_ctx, _in1, _in2, _out1, _out2| Ok(())); register_task!(s, "task5", [Is Os], task1); register_task!(s, "task6", [I O Is Os], - |_ctx, _i1: &DataInstance, _o1: &mut Output, _is: &[DataInstance], _os: &mut [Output]| Ok(())); + |_ctx, _i1: &DataInstance, _o1: &mut Output, _is: &[DataInstance], + _os: &mut [Output]| Ok(())); } fn task_cat(_ctx: &mut Context, inputs: &[DataInstance], outputs: &mut [Output]) -> TaskResult<()> { @@ -264,25 +265,27 @@ fn run_cat_task() { fn run_long_cat() { let (mut s, handle) = setup( "run_long_cat", - vec![call_msg( - 1, - "run_long_cat/cat", - vec![ - data_spec( - 1, - "in1", - Some(DataLocation::Memory( - [0u8; MEM_BACKED_LIMIT - 1].as_ref().into(), - )), - ), - data_spec( - 2, - "in2", - Some(DataLocation::Memory([0u8; 2].as_ref().into())), - ), - ], - vec![data_spec(6, "out", None)], - )], + vec![ + call_msg( + 1, + "run_long_cat/cat", + vec![ + data_spec( + 1, + "in1", + Some(DataLocation::Memory( + [0u8; MEM_BACKED_LIMIT - 1].as_ref().into(), + )), + ), + data_spec( + 2, + "in2", + Some(DataLocation::Memory([0u8; 2].as_ref().into())), + ), + ], + vec![data_spec(6, "out", None)], + ), + ], ); s.register_task("cat", task_cat); s.run(); @@ -301,12 +304,14 @@ fn run_long_cat() { fn run_empty_cat() { let (mut s, handle) = setup( "run_empty_cat", - vec![call_msg( - 1, - "run_empty_cat/cat", - vec![], - vec![data_spec(3, "out", None)], - )], + vec![ + call_msg( + 1, + "run_empty_cat/cat", + vec![], + vec![data_spec(3, "out", None)], + ), + ], ); s.register_task("cat", task_cat); s.run(); @@ -322,16 +327,16 @@ fn run_empty_cat() { fn run_pass_cat() { let (mut s, handle) = setup( "run_pass_cat", - vec![call_msg( - 2, - "run_pass_cat/cat", - vec![data_spec( - 1, - "in", - Some(DataLocation::Memory("drip".into())), - )], - vec![data_spec(2, "out", None)], - )], + vec![ + call_msg( + 2, + "run_pass_cat/cat", + vec![ + data_spec(1, "in", Some(DataLocation::Memory("drip".into()))), + ], + vec![data_spec(2, "out", None)], + ), + ], ); s.register_task("cat", task_cat); s.run(); @@ -347,12 +352,14 @@ fn run_pass_cat() { fn test_make_file_backed() { let (mut s, handle) = setup( "test_make_file_backed", - vec![call_msg( - 2, - "test_make_file_backed/mfb", - vec![], - vec![data_spec(3, "out", None)], - )], + vec![ + call_msg( + 2, + "test_make_file_backed/mfb", + vec![], + vec![data_spec(3, "out", None)], + ), + ], ); s.register_task("mfb", |_ctx, _ins, outs| { write!(outs[0], "Rainfall")?; @@ -373,16 +380,16 @@ fn test_make_file_backed() { fn test_get_path_writing() { let (mut s, handle) = setup( "test_get_path_writing", - vec![call_msg( - 2, - "test_get_path_writing/gp", - vec![data_spec( - 1, - "in", - Some(DataLocation::Memory("drizzle".into())), - )], - vec![], - )], + vec![ + call_msg( + 2, + "test_get_path_writing/gp", + vec![ + data_spec(1, "in", Some(DataLocation::Memory("drizzle".into()))), + ], + vec![], + ), + ], ); s.register_task("gp", |_ctx, ins, _outs| { let p = ins[0].get_path(); @@ -404,12 +411,14 @@ fn test_get_path_writing() { fn run_stage_file() { let (mut s, handle) = setup( "run_stage_file", - vec![call_msg( - 2, - "run_stage_file/stage", - vec![], - vec![data_spec(2, "out", None)], - )], + vec![ + call_msg( + 2, + "run_stage_file/stage", + vec![], + vec![data_spec(2, "out", None)], + ), + ], ); s.register_task("stage", |_ctx, _inp, outp| { let mut f = fs::File::create("testfile.txt").unwrap(); @@ -494,11 +503,13 @@ fn read_set_content_type() { let mut call = call_msg( 2, "read_set_content_type/foo", - vec![data_spec( - 2, - "out", - Some(DataLocation::Memory((b"content!" as &[u8]).into())), - )], + vec![ + data_spec( + 2, + "out", + Some(DataLocation::Memory((b"content!" as &[u8]).into())), + ), + ], vec![data_spec(3, "out", None)], ); call.inputs[0].info = Some(ObjectInfo { diff --git a/src/client/rpc.rs b/src/client/rpc.rs deleted file mode 100644 index 082fc12..0000000 --- a/src/client/rpc.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::task::Task; -use client::dataobject::DataObject; -use client::task::TaskInput; -use common::convert::ToCapnp; -use common::id::DataObjectId; - -macro_rules! to_capnp_list { - ($builder:expr, $items:expr, $name:ident) => { - { - let mut builder = $builder.$name($items.len() as u32); - for (i, obj) in $items.iter().enumerate() { - obj.to_capnp(&mut builder.reborrow().get(i as u32)); - } - } - } -} -macro_rules! from_capnp_list { - ($builder:expr, $items:ident, $obj:ident) => { - { - $builder - .$items()? - .iter() - .map(|item| $obj::from_capnp(&item)) - .collect() - } - } -} - -impl<'a> ToCapnp<'a> for TaskInput { - type Builder = ::client_capnp::task::in_data_object::Builder<'a>; - - fn to_capnp(&self, builder: &mut Self::Builder) { - self.data_object - .id - .to_capnp(&mut builder.reborrow().get_id().unwrap()); - - if let &Some(ref label) = &self.label { - builder.reborrow().set_label(label); - } - } -} - -impl<'a> ToCapnp<'a> for Task { - type Builder = ::client_capnp::task::Builder<'a>; - - fn to_capnp(&self, builder: &mut Self::Builder) { - self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); - builder.set_task_type(&self.command); - - to_capnp_list!(builder.reborrow(), self.inputs, init_inputs); - to_capnp_list!( - builder.reborrow(), - self.outputs - .iter() - .map(|o| o.id) - .collect::>(), - init_outputs - ); - self.attributes - .to_capnp(&mut builder.reborrow().get_attributes().unwrap()); - } -} -impl<'a> ToCapnp<'a> for DataObject { - type Builder = ::client_capnp::data_object::Builder<'a>; - - fn to_capnp(&self, builder: &mut Self::Builder) { - self.id.to_capnp(&mut builder.reborrow().get_id().unwrap()); - builder.set_keep(self.keep.get()); - builder.set_label(&self.label); - builder.set_data_type(self.data_type.to_capnp()); - - if let &Some(ref data) = &self.data { - builder.set_data(&data); - builder.set_has_data(true); - } else { - builder.set_has_data(false); - } - - self.attributes - .to_capnp(&mut builder.reborrow().get_attributes().unwrap()); - } -} diff --git a/src/client/task.rs b/src/client/task.rs deleted file mode 100644 index a3342a0..0000000 --- a/src/client/task.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::error::Error; -use common::Attributes; -use common::id::TaskId; -use super::session::DataObjectPtr; - -pub struct TaskInput { - pub label: Option, - pub data_object: DataObjectPtr, -} - -pub struct Task { - pub id: TaskId, - pub command: String, - pub inputs: Vec, - pub outputs: Vec, - pub attributes: Attributes, -} - -impl Task { - pub fn output(&self) -> Result> { - if self.outputs.len() == 1 { - return Ok(self.outputs[0].clone()); - } - - bail!("There is not a single output") - } -} diff --git a/src/start/localcluster.rs b/src/start/localcluster.rs deleted file mode 100644 index 9fc0751..0000000 --- a/src/start/localcluster.rs +++ /dev/null @@ -1,55 +0,0 @@ -use start::starter::{Starter, StarterConfig}; -use std::net::SocketAddr; -use std::error::Error; -use client::client::Client; -use start::common::{default_logging_directory, ensure_directory}; - -pub struct LocalCluster { - listen_addr: SocketAddr, - starter: Starter, -} - -impl LocalCluster { - pub fn new( - worker_cpus: Vec>, - listen_port: u16, - http_port: u16, - ) -> Result> { - let listen_addr = SocketAddr::new("127.0.0.1".parse()?, listen_port); - let http_addr = SocketAddr::new("127.0.0.1".parse()?, http_port); - - let log_dir = default_logging_directory("rain"); - ensure_directory(&log_dir, "logging directory")?; - - let config = StarterConfig::new( - worker_cpus, - listen_addr, - http_addr, - &log_dir, - "".to_owned(), - false, - vec![], - ); - - let mut cluster = LocalCluster { - listen_addr, - starter: Starter::new(config), - }; - cluster.starter.start()?; - - Ok(cluster) - } - pub fn new_simple(listen_port: u16, http_port: u16) -> Result> { - Self::new(vec![None], listen_port, http_port) - } - - pub fn create_client(&self) -> Result> { - Client::new(self.listen_addr) - } -} - -impl Drop for LocalCluster { - fn drop(&mut self) { - self.starter.kill_all(); - } -} From 163ad3b584b0a83c9dae8bcca5e965bf8d2fe9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sat, 7 Jul 2018 23:05:04 +0200 Subject: [PATCH 18/18] [api] add client test module, remove test crate --- Cargo.lock | 7 ---- Cargo.toml | 1 - rain_client/src/client/client.rs | 44 ++++++++++++++++++++++++++ rain_client/src/client/localcluster.rs | 11 +++---- rain_client/src/client/mod.rs | 4 +-- rain_client/src/client/session.rs | 3 +- rain_client_test/Cargo.toml | 10 ------ rain_client_test/src/main.rs | 27 ---------------- 8 files changed, 53 insertions(+), 54 deletions(-) delete mode 100644 rain_client_test/Cargo.toml delete mode 100644 rain_client_test/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 28716e9..3ccb132 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -569,13 +569,6 @@ dependencies = [ "tokio-core 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rain_client_test" -version = "0.3.0" -dependencies = [ - "rain_client 0.3.0", -] - [[package]] name = "rain_core" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 047ef1c..516b1a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,6 @@ version = "0.3.0" members = [ "rain_client", - "rain_client_test", "rain_core", "rain_server", "rain_task", diff --git a/rain_client/src/client/client.rs b/rain_client/src/client/client.rs index 267af9c..7f86e86 100644 --- a/rain_client/src/client/client.rs +++ b/rain_client/src/client/client.rs @@ -27,3 +27,47 @@ impl Client { } } +#[cfg(test)] +mod tests { + use super::super::localcluster::LocalCluster; + use super::super::tasks::CommonTasks; + use super::Client; + use super::Session; + use std::env; + + #[allow(dead_code)] + struct TestContext { + cluster: LocalCluster, + client: Client, + session: Session, + } + + fn ctx() -> TestContext { + let rain = env::var("RAIN_BINARY").unwrap(); + + let cluster = LocalCluster::new(&rain).unwrap(); + let client = cluster.create_client().unwrap(); + let session = client.new_session().unwrap(); + + TestContext { + cluster, + client, + session, + } + } + + #[test] + fn concat() { + let mut ctx = ctx(); + let a = ctx.session.blob(vec![1, 2, 3]); + let b = ctx.session.blob(vec![4, 5, 6]); + let c = ctx.session.concat(&[a, b]); + c.output().keep(); + ctx.session.submit().unwrap(); + ctx.session.wait(&[c.clone()], &[]).unwrap(); + assert_eq!( + ctx.session.fetch(&c.output()).unwrap(), + vec![1, 2, 3, 4, 5, 6] + ); + } +} diff --git a/rain_client/src/client/localcluster.rs b/rain_client/src/client/localcluster.rs index 01c92c4..45e1b11 100644 --- a/rain_client/src/client/localcluster.rs +++ b/rain_client/src/client/localcluster.rs @@ -1,21 +1,19 @@ +use super::client::Client; use std::error::Error; use std::net::SocketAddr; -use super::client::Client; -use std::process::{Child, Command}; use std::path::PathBuf; -use std::{thread, time}; -use std::process::Stdio; +use std::process::{Command, Stdio}; pub struct LocalCluster { listen_addr: SocketAddr, - binary: PathBuf + binary: PathBuf, } impl LocalCluster { pub fn new(binary: &str) -> Result> { let mut cluster = LocalCluster { binary: PathBuf::from(binary), - listen_addr: SocketAddr::new("127.0.0.1".parse()?, 7210) + listen_addr: SocketAddr::new("127.0.0.1".parse()?, 7210), }; cluster.start()?; @@ -43,6 +41,7 @@ impl LocalCluster { } impl Drop for LocalCluster { + #![allow(unused_must_use)] fn drop(&mut self) { Client::new(self.listen_addr).unwrap().terminate_server(); } diff --git a/rain_client/src/client/mod.rs b/rain_client/src/client/mod.rs index d40fe1d..1dc689a 100644 --- a/rain_client/src/client/mod.rs +++ b/rain_client/src/client/mod.rs @@ -1,7 +1,7 @@ pub mod client; +pub mod localcluster; pub mod session; pub mod tasks; -pub mod localcluster; #[macro_use] mod rpc; @@ -10,5 +10,5 @@ mod dataobject; mod task; pub use self::client::Client; -pub use self::session::Session; pub use self::localcluster::LocalCluster; +pub use self::session::Session; diff --git a/rain_client/src/client/session.rs b/rain_client/src/client/session.rs index 9097f07..aae30f1 100644 --- a/rain_client/src/client/session.rs +++ b/rain_client/src/client/session.rs @@ -160,8 +160,9 @@ impl Session { } impl Drop for Session { + #![allow(unused_must_use)] fn drop(&mut self) { - self.comm.close_session(self.id).unwrap(); debug!("Session {} destroyed", self.id); + self.comm.close_session(self.id); } } diff --git a/rain_client_test/Cargo.toml b/rain_client_test/Cargo.toml deleted file mode 100644 index 57ebf91..0000000 --- a/rain_client_test/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "rain_client_test" -version = "0.3.0" -authors = ["Jakub Beranek "] -description = "A testing Rain client in Rust." -license = "MIT" -publish = false - -[dependencies] -rain_client = "0.3.0" diff --git a/rain_client_test/src/main.rs b/rain_client_test/src/main.rs deleted file mode 100644 index d12eef0..0000000 --- a/rain_client_test/src/main.rs +++ /dev/null @@ -1,27 +0,0 @@ -extern crate rain_client; - -use std::error::Error; -use rain_client::tasks::CommonTasks; -use rain_client::client::LocalCluster; - -fn test() -> Result<(), Box> { - let cluster = LocalCluster::new("/home/kobzol/projects/it4i/rain/target/debug/rain")?; - - let client = cluster.create_client()?; - let mut s = client.new_session()?; - - let a = s.open("/tmp/asd.txt".to_owned()); - let b = s.open("/tmp/asd2.txt".to_owned()); - - let c = s.concat(&[a.output(), b.output()]); - c.output().keep(); - s.submit()?; - let res = s.fetch(&c.output())?; - println!("{}", String::from_utf8(res)?); - - Ok(()) -} - -fn main() { - test().unwrap(); -}