diff --git a/DESCRIPTION b/DESCRIPTION index bbf845720..8a11d060f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: mirai Type: Package Title: Minimalist Async Evaluation Framework for R -Version: 1.1.0.9009 +Version: 1.1.0.9010 Description: High-performance parallel code execution and distributed computing. Designed for simplicity, a 'mirai' evaluates an R expression asynchronously, on local or network resources, resolving automatically upon completion. @@ -31,4 +31,4 @@ Suggests: knitr, markdown VignetteBuilder: knitr -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 diff --git a/NEWS.md b/NEWS.md index 041164194..77599cbb9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ -# mirai 1.1.0.9009 (development) +# mirai 1.1.0.9010 (development) -* `serialization()` function signature simplified for clarity and ease of use. +* `serialization()` function signature and return value slightly modified for clarify. Successful registration / cancellation mesasges are no longer printed to the console. * `dispatcher()` argument 'retry' now defaults to FALSE for consistency with non-dispatcher behaviour. * `remote_config()` gains argument 'quote' to control whether or not to quote the daemon launch commmand, and now works with Slurm (thanks @michaelmayer2 #119). * Ephemeral daemons now exit as soon as permissible, eiliminating the 2s linger period. diff --git a/R/daemons.R b/R/daemons.R index 2291ea903..3bbd5d768 100644 --- a/R/daemons.R +++ b/R/daemons.R @@ -489,41 +489,34 @@ status <- function(.compute = "default") { #' Registers custom serialization and unserialization functions for sending and #' receiving reference objects. #' +#' @param fns \strong{either} a list comprising 2 functions: \cr serialization +#' function: must accept a reference object (or list of objects) inheriting +#' from \sQuote{class} and return a raw vector.\cr unserialization function: +#' must accept a raw vector and return a reference object (or list of +#' reference objects).\cr \strong{or else} NULL to reset. #' @param class the class of reference object (as a character string) that these -#' functions are applied to, e.g. 'ArrowTabular' or 'torch_tensor', -#' \strong{or else} NULL to cancel registered functions. -#' @param sfunc serialization function: must accept a reference object (or list -#' of objects) inheriting from \sQuote{class} and return a raw vector. -#' @param ufunc unserialization function: must accept a raw vector and return -#' a reference object (or list of reference objects). +#' functions are applied to, e.g. 'ArrowTabular' or 'torch_tensor'. #' @param vec [default FALSE] if FALSE the functions must accept and return #' reference objects individually e.g. \code{arrow::write_to_raw} and #' \code{arrow::read_ipc_stream}. If TRUE, the functions are vectorized and #' must accept and return a list of reference objects, e.g. #' \code{torch::torch_serialize} and \code{torch::torch_load}. #' -#' @return Invisibly, a list comprising the currently-registered values for -#' 'class', 'sfunc', 'ufunc' and 'vec', or else NULL if unregistered. A -#' message is printed to the console when functions are successfully -#' registered or reset. +#' @return Invisibly, a list comprising 'fns', class', and 'vec', or else NULL +#' if supplied to 'fns'. #' -#' @details Registering new functions replaces any existing registered -#' functions. -#' -#' To cancel registered functions, specify 'class' as NULL, without the -#' need to supply 'sfunc' or 'ufunc'. -#' -#' Calling without any arguments returns the pairlist of -#' currently-registered serialization functions. +#' @details Registering new functions replaces any existing registered functions. #' #' This function may be called prior to or after setting daemons, with the #' registered functions applying across all compute profiles. #' +#' Calling without any arguments returns a list comprising the registered +#' values for 'fns', class', and 'vec', or else NULL if not registered. +#' #' @examples #' reg <- serialization( -#' class = "", -#' sfunc = function(x) serialize(x, NULL), -#' ufunc = base::unserialize +#' list(function(x) serialize(x, NULL), base::unserialize), +#' class = "example_class" #' ) #' reg #' @@ -532,24 +525,23 @@ status <- function(.compute = "default") { #' #' @export #' -serialization <- function(class, sfunc, ufunc, vec = FALSE) { +serialization <- function(fns, class, vec = FALSE) { - missing(class) && return(.[["serial"]]) + missing(fns) && return(.[["serial"]]) - if (is.null(class)) { + if (is.null(fns)) { serial <- NULL next_config(NULL) - cat("mirai serialization functions cancelled\n", file = stderr()) - } else if (is.character(class) && is.function(sfunc) && is.function(ufunc)) { - serial <- list(class, sfunc, ufunc, vec) - next_config(refhook = list(sfunc, ufunc), class = class, vec = vec) - cat("mirai serialization functions registered\n", file = stderr()) + } else if (length(fns) == 2L && is.function(fns[[1L]]) && is.function(fns[[2L]])) { + is.character(class) || stop(._[["character_class"]]) + serial <- list(fns, class, vec) + next_config(fns, class = class, vec = vec) } else { stop(._[["serial_invalid"]]) } `[[<-`(., "serial", serial) - register_everywhere(serial = serial) + register_everywhere(serial) invisible(serial) } @@ -673,12 +665,12 @@ query_status <- function(envir) { register_everywhere <- function(serial) for (.compute in names(..)) everywhere( - mirai::serialization(class = serial[[1L]], sfunc = serial[[2L]], ufunc = serial[[3L]], vec = serial[[4L]]), + mirai::serialization(serial[[1L]], class = serial[[2L]], vec = serial[[3L]]), .args = list(serial = serial), .compute = .compute ) check_register_everywhere <- function(serial = .[["serial"]]) - if (length(serial[[1L]])) register_everywhere(serial = serial) + if (length(serial[[1L]])) register_everywhere(serial) ._scm_. <- as.raw(c(0x07, 0x00, 0x00, 0x00, 0x42, 0x0a, 0x03, 0x00, 0x00, 0x00, 0x02, 0x03, 0x04, 0x00, 0x00, 0x05, 0x03, 0x00, 0x05, 0x00, 0x00, 0x00, 0x55, 0x54, 0x46, 0x2d, 0x38, 0xfc, 0x00, 0x00, 0x00)) diff --git a/R/mirai-package.R b/R/mirai-package.R index 990e79202..90a5b7d2e 100644 --- a/R/mirai-package.R +++ b/R/mirai-package.R @@ -85,6 +85,7 @@ ._ <- list2env( list( arglen = "'args' and/or 'url' must be of length 1 or the same length", + character_class = "'class' must be a character string", cluster_inactive = "cluster is no longer active", correct_context = "'host' must be specified if not using directly in a function argument", daemons_unset = "a numeric value for 'url' requires daemons to be set", @@ -98,7 +99,7 @@ register_cluster = "this function requires a more recent version of R", requires_daemons = "launching one local daemon as none previously set", requires_local = "SSH tunnelling requires 'url' hostname to be '127.0.0.1' or 'localhost'", - serial_invalid = "'class' must be a character value or NULL, 'sfunc' and 'ufunc' must be functions", + serial_invalid = "'fns' must be a list of 2 functions or NULL", single_url = "only one 'url' should be specified", sync_timeout = "initial sync with dispatcher/daemon timed out after 10s", url_spec = "numeric value for 'url' is out of bounds", diff --git a/README.md b/README.md index e2b15e37c..aafd31b95 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ method: ``` r m[] -#> [1] 50.10154 +#> [1] 47.76782 ``` It is not necessary to wait, as the mirai resolves automatically @@ -94,7 +94,7 @@ available at `$data`. m #> < mirai [$data] > m$data -#> [1] 50.10154 +#> [1] 47.76782 ``` ### Daemons diff --git a/man/serialization.Rd b/man/serialization.Rd index 2e29a038c..b210a7304 100644 --- a/man/serialization.Rd +++ b/man/serialization.Rd @@ -4,18 +4,17 @@ \alias{serialization} \title{Custom Serialization Functions} \usage{ -serialization(class, sfunc, ufunc, vec = FALSE) +serialization(fns, class, vec = FALSE) } \arguments{ -\item{class}{the class of reference object (as a character string) that these -functions are applied to, e.g. 'ArrowTabular' or 'torch_tensor', -\strong{or else} NULL to cancel registered functions.} - -\item{sfunc}{serialization function: must accept a reference object (or list -of objects) inheriting from \sQuote{class} and return a raw vector.} +\item{fns}{\strong{either} a list comprising 2 functions: \cr serialization +function: must accept a reference object (or list of objects) inheriting +from \sQuote{class} and return a raw vector.\cr unserialization function: +must accept a raw vector and return a reference object (or list of +reference objects).\cr \strong{or else} NULL to reset.} -\item{ufunc}{unserialization function: must accept a raw vector and return -a reference object (or list of reference objects).} +\item{class}{the class of reference object (as a character string) that these +functions are applied to, e.g. 'ArrowTabular' or 'torch_tensor'.} \item{vec}{[default FALSE] if FALSE the functions must accept and return reference objects individually e.g. \code{arrow::write_to_raw} and @@ -24,33 +23,26 @@ must accept and return a list of reference objects, e.g. \code{torch::torch_serialize} and \code{torch::torch_load}.} } \value{ -Invisibly, a list comprising the currently-registered values for - 'class', 'sfunc', 'ufunc' and 'vec', or else NULL if unregistered. A - message is printed to the console when functions are successfully - registered or reset. +Invisibly, a list comprising 'fns', class', and 'vec', or else NULL + if supplied to 'fns'. } \description{ Registers custom serialization and unserialization functions for sending and receiving reference objects. } \details{ -Registering new functions replaces any existing registered - functions. - - To cancel registered functions, specify 'class' as NULL, without the - need to supply 'sfunc' or 'ufunc'. - - Calling without any arguments returns the pairlist of - currently-registered serialization functions. +Registering new functions replaces any existing registered functions. This function may be called prior to or after setting daemons, with the registered functions applying across all compute profiles. + + Calling without any arguments returns a list comprising the registered + values for 'fns', class', and 'vec', or else NULL if not registered. } \examples{ reg <- serialization( - class = "", - sfunc = function(x) serialize(x, NULL), - ufunc = base::unserialize + list(function(x) serialize(x, NULL), base::unserialize), + class = "example_class" ) reg diff --git a/tests/tests.R b/tests/tests.R index 11c9a12f9..8df0a6789 100644 --- a/tests/tests.R +++ b/tests/tests.R @@ -29,7 +29,8 @@ nanotesterr(launch_remote(c("tcp://localhost:5555", "tcp://localhost:6666", "tcp nanotesterr(launch_local(1L), "requires daemons to be set") nanotestn(everywhere(mirai::serialization())) nanotestn(serialization()) -nanotesterr(serialization(list(NULL)), "must be a character value or NULL") +nanotesterr(serialization(list(NULL)), "must be a list of 2 functions or NULL") +nanotesterr(serialization(list(identity, identity), class = NA), "must be a character string") nanotest(is.character(host_url())) nanotest(substr(host_url(ws = TRUE, tls = TRUE), 1L, 3L) == "wss") nanotest(substr(host_url(tls = TRUE), 1L, 3L) == "tls") @@ -240,8 +241,8 @@ connection && .Platform[["OS.type"]] != "windows" && Sys.getenv("NOT_CRAN") == " nanotestz(sum(tstatus[, "assigned"])) nanotestz(sum(tstatus[, "complete"])) nanotestz(daemons(0)) - nanotest(is.list(serialization(class = "", sfunc = function(x) serialize(x, NULL), ufunc = unserialize))) - nanotest(is.function(serialization()[[2L]])) + nanotest(is.list(serialization(fns = list(function(x) serialize(x, NULL), unserialize), class = "tst_cls"))) + nanotest(is.function(serialization()[[1L]][[2L]])) nanotesto(daemons(url = "wss://127.0.0.1:0", token = TRUE, pass = "test")) nanotestn(launch_local(1L)) Sys.sleep(1L) diff --git a/vignettes/databases.Rmd b/vignettes/databases.Rmd index 09dd5a799..c71bf2ce2 100644 --- a/vignettes/databases.Rmd +++ b/vignettes/databases.Rmd @@ -69,9 +69,11 @@ everywhere({ con <<- dbConnect(adbi::adbi("adbcsqlite"), uri = ":memory:") }) serialization( - class = "nanoarrow_array_stream", - sfunc = arrow::write_to_raw, - ufunc = function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + list( + arrow::write_to_raw, + function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + ), + class = "nanoarrow_array_stream" ) ``` `mirai()` calls may then be used to write to or query the database all in the Arrow format. @@ -249,9 +251,8 @@ server <- function(input, output, session) { # serialization() specifies the native Arrow serialization functions serialization( - class = "nanoarrow_array_stream", - sfunc = arrow::write_to_raw, - ufunc = nanoarrow::read_nanoarrow + list(arrow::write_to_raw, nanoarrow::read_nanoarrow), + class = "nanoarrow_array_stream" ) # run Shiny app diff --git a/vignettes/databases.Rmd.orig b/vignettes/databases.Rmd.orig index aaafef49d..f47373a98 100644 --- a/vignettes/databases.Rmd.orig +++ b/vignettes/databases.Rmd.orig @@ -60,9 +60,11 @@ everywhere({ con <<- dbConnect(adbi::adbi("adbcsqlite"), uri = ":memory:") }) serialization( - class = "nanoarrow_array_stream", - sfunc = arrow::write_to_raw, - ufunc = function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + list( + arrow::write_to_raw, + function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + ), + class = "nanoarrow_array_stream" ) ``` `mirai()` calls may then be used to write to or query the database all in the Arrow format. @@ -182,9 +184,8 @@ server <- function(input, output, session) { # serialization() specifies the native Arrow serialization functions serialization( - class = "nanoarrow_array_stream", - sfunc = arrow::write_to_raw, - ufunc = nanoarrow::read_nanoarrow + list(arrow::write_to_raw, nanoarrow::read_nanoarrow), + class = "nanoarrow_array_stream" ) # run Shiny app diff --git a/vignettes/mirai.Rmd b/vignettes/mirai.Rmd index 83a2470c0..ee8ad5e60 100644 --- a/vignettes/mirai.Rmd +++ b/vignettes/mirai.Rmd @@ -65,7 +65,7 @@ To wait for and collect the evaluated result, use the mirai's `[]` method: ``` r m[] -#> [1] 57.81648 +#> [1] 58.47676 ``` It is not necessary to wait, as the mirai resolves automatically whenever the async operation completes, the evaluated result then available at `$data`. @@ -74,7 +74,7 @@ It is not necessary to wait, as the mirai resolves automatically whenever the as m #> < mirai [$data] > m$data -#> [1] 57.81648 +#> [1] 58.47676 ``` For easy programmatic use of `mirai()`, '.expr' accepts a pre-constructed language object, and also a list of named arguments passed via '.args'. So, the following would be equivalent to the above: @@ -89,7 +89,7 @@ args <- list(mean = input$x, sd = input$y) m <- mirai(.expr = expr, .args = args) m[] -#> [1] 57.51712 +#> [1] 58.44681 ``` [« Back to ToC](#table-of-contents) @@ -169,8 +169,8 @@ for (i in 1:10) { #> iteration 4 successful #> iteration 5 successful #> iteration 6 successful -#> Error: random error #> iteration 7 successful +#> Error: random error #> iteration 8 successful #> iteration 9 successful #> iteration 10 successful @@ -205,12 +205,12 @@ status() #> #> $daemons #> i online instance assigned complete -#> abstract://8acde0f178041fbbcace7910 1 1 1 1 0 -#> abstract://8c35ac7da595775a3bb6609f 2 1 1 1 0 -#> abstract://1cfc035476c41bd5ba3b468c 3 1 1 1 0 -#> abstract://30759a91108c9bd0ae116c98 4 1 1 1 0 -#> abstract://a25bcabc8ec8c52739d0509a 5 1 1 1 0 -#> abstract://e4098f70b7008941abe4ce34 6 1 1 1 0 +#> abstract://97c9b9931341ea8a4b70b6c8 1 1 1 0 0 +#> abstract://5e216a9efcd8d5a15e5eb4a9 2 1 1 0 0 +#> abstract://3d01bf6f661fb4382a6683d6 3 1 1 0 0 +#> abstract://c91ef6028dfc652329f4c159 4 1 1 0 0 +#> abstract://f56375e4bc132cad42a40a01 5 1 1 0 0 +#> abstract://21f4692550e10ab4e2f233b4 6 1 1 0 0 ``` The default `dispatcher = TRUE` creates a `dispatcher()` background process that connects to individual daemon processes on the local machine. This ensures that tasks are dispatched efficiently on a first-in first-out (FIFO) basis to daemons for processing. Tasks are queued at the dispatcher and sent to a daemon as soon as it can accept the task for immediate execution. @@ -239,7 +239,7 @@ status() #> [1] 6 #> #> $daemons -#> [1] "abstract://4de09472768195a889a42707" +#> [1] "abstract://df872fae5724c6e97d38f77e" ``` This implementation sends tasks immediately, and ensures that tasks are evenly-distributed amongst daemons. This means that optimal scheduling is not guaranteed as the duration of tasks cannot be known *a priori*. As an example, tasks could be queued at a daemon behind a long-running task, whilst other daemons are idle having already completed their tasks. @@ -265,14 +265,14 @@ By super-assignment, the conenction 'con' will be available in the global enviro ``` r m <- mirai(capture.output(str(con))) m[] -#> [1] "Formal class 'SQLiteConnection' [package \"RSQLite\"] with 8 slots" -#> [2] " ..@ ptr : " -#> [3] " ..@ dbname : chr \"/tmp/RtmpCSqxZK/file141403163fbe6\"" -#> [4] " ..@ loadable.extensions: logi TRUE" -#> [5] " ..@ flags : int 70" -#> [6] " ..@ vfs : chr \"\"" -#> [7] " ..@ ref : " -#> [8] " ..@ bigint : chr \"integer64\"" +#> [1] "Formal class 'SQLiteConnection' [package \"RSQLite\"] with 8 slots" +#> [2] " ..@ ptr : " +#> [3] " ..@ dbname : chr \"/tmp/RtmpD0gU3W/file5bbd58866397\"" +#> [4] " ..@ loadable.extensions: logi TRUE" +#> [5] " ..@ flags : int 70" +#> [6] " ..@ vfs : chr \"\"" +#> [7] " ..@ ref : " +#> [8] " ..@ bigint : chr \"integer64\"" #> [9] " ..@ extended_types : logi FALSE" ``` Disconnect from the database everywhere, and set the number of daemons to zero to reset. @@ -362,7 +362,7 @@ By specifying `dispatcher = FALSE`, remote daemons connect directly to the host ``` r daemons(url = host_url(), dispatcher = FALSE) -#> [1] "tcp://hostname:36153" +#> [1] "tcp://hostname:42563" ``` Note that above, calling `host_url()` without a port value uses the default of '0'. This is a wildcard value that will automatically cause a free ephemeral port to be assigned. The actual assigned port is provided in the return value of the call, or it may be queried at any time via `status()`. @@ -376,7 +376,7 @@ status() #> [1] 0 #> #> $daemons -#> [1] "tcp://hostname:36153" +#> [1] "tcp://hostname:42563" ``` To reset all connections and revert to default behaviour: @@ -463,10 +463,10 @@ daemons(n = 2, url = host_url()) #> [1] 2 launch_remote(1:2) #> [1] -#> Rscript -e 'mirai::daemon("tcp://hostname:34147",rs=c(10407,1840327960,54744121,1550472870,-1103900433,-1039568284,1321446933))' +#> Rscript -e 'mirai::daemon("tcp://hostname:35767",rs=c(10407,759990225,-1325441186,330638407,523020252,-484141267,1462196234))' #> #> [2] -#> Rscript -e 'mirai::daemon("tcp://hostname:46357",rs=c(10407,-35807015,-1998252114,-991467071,-1052451096,-1759681021,619158771))' +#> Rscript -e 'mirai::daemon("tcp://hostname:43679",rs=c(10407,-714514434,-395272110,69901107,-741022113,1467565944,-1430404455))' daemons(0) #> [1] 0 ``` @@ -493,37 +493,37 @@ The generated self-signed certificate is available via `launch_remote()`. This f ``` r launch_remote(1) #> [1] -#> Rscript -e 'mirai::daemon("wss://hostname:35807/1",tls=c("-----BEGIN CERTIFICATE----- +#> Rscript -e 'mirai::daemon("wss://hostname:40215/1",tls=c("-----BEGIN CERTIFICATE----- #> MIIFNzCCAx+gAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMREwDwYDVQQDDAhrdW1h #> bW90bzERMA8GA1UECgwITmFub25leHQxCzAJBgNVBAYTAkpQMB4XDTAxMDEwMTAw #> MDAwMFoXDTMwMTIzMTIzNTk1OVowMzERMA8GA1UEAwwIa3VtYW1vdG8xETAPBgNV #> BAoMCE5hbm9uZXh0MQswCQYDVQQGEwJKUDCCAiIwDQYJKoZIhvcNAQEBBQADggIP -#> ADCCAgoCggIBAOZRT7AKu9ziWM2xJC3bXAoBWRSBb7p22tka5G4h535RU9vh8nfJ -#> jHhhGbrfpLwao406P2mf33/WSTlB1GSnpFC14G0X+/LjtLsNOkBY6kakjyVbsC3S -#> Qe6C1ukJkvztGyiMK3NzCLqSaWjwfeyRxu90Va0NPLLQq8E+Lo8IFnMUC7UOXYO6 -#> chg0AmbV9vj92H73Qs1MSGy2UO1y3AkHnZxZb/ZjRxLZvGbGQuY6EXxn8y6EZVJj -#> GL1XZK2dr+ncGuFeJxQnVRLffQXWA2/MFmv3R3flk6XL5X0Q1TLNjF6Vcelm2L8W -#> PMT+rEAi7aIn81Rd0yakOJrIkzXwAlrIpZsYnl6CO/BsB1hKLWAUkCIbP260nXYV -#> x4zVPkpbWgY+cQhRqZsi6n5pnwfYxnIBMOYAAWKlpW/RDhEfggBQD9rXoBm9b0uV -#> 9Bxl1dIpiGyWB0HyYkIZdx7G//wNG+vCleUbt3f5f71orp7OcoUoJ5oB3IjzSmWf -#> uG3TIObTsDnW9bohQ0CRYvUGSc/yPysC25DqzOOuX8hrQ+Qng/jxYIPhyaLf23ri -#> t1eRNhfeB5EvMkehfLC0Yz5nV9Q173jsJ8h+vCuqDop9Dx2Xb0N2wIVZqYsuW4hN -#> fWT/Hz/Oo5VMB4C1ZHOgfig6EcEtMdnpiZfuCfv1bEABwpJH3GecSr4bAgMBAAGj -#> VjBUMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFCZO/cLxyiDYGEsVTZ4u -#> MbyFc2+hMB8GA1UdIwQYMBaAFCZO/cLxyiDYGEsVTZ4uMbyFc2+hMA0GCSqGSIb3 -#> DQEBCwUAA4ICAQCuv3Q+Owk0GEJo777s0X5VM7CrfCQVOggwl7pnXiRYqZk3DzbX -#> huCbI0KDbanmxDzUjWEkCq+85rzFCP5NZY4Hki0/fdGOkyiO0ArrwLEwZ5QZ7aL4 -#> eaP5YEwkelzYhr88BZMURKDPpA0fTScLuXqP+lw2ZyMdH85cGa1a0CzTl8CVtw37 -#> P7JxbburW08z8Owud2iFPL13MapbrNyRARdgw/jCeOn/Pzr4SHFOIbfEaNztKjUr -#> vEoZpGroqXe5v7T+AtxSrTYW5NU5dgCSXiJjJEpWF1gVy8t+ZpPo8v5Szg9QudQB -#> dsS9dZRKZo4oEpDgn/jykI6C7xfRrjxViGKHSAmJP1KuWrU6CVZtAAEMIJvpwaAM -#> 0Yke+C3yVPVT88V50hLAigrbaingOcTT+g5a6V8wSSzYkjgaBj9SWl1K3+orZ6B4 -#> VSU4DNKkFh/yQTQBQ861GFJVIFcSV4SvKSEq8lwXjpwg1qHmoj9r7qxcN8+sGZsx -#> RYKqk5XfXXSStmC8ldVP0opVp3RwWkRVxaERItGbbGxEfVRiXwoL/MQPwtQUaTVI -#> zP0XFhVY3NVfLEwsv4R+QXJiCcjAbTdBtrMNePb+TqkX/+ksNq/6kyPrbxtsVLNj -#> LtA7qzn3+5oQFjXaFqltkWY1O1Xt/cc8DK5oYYhbzVt5h1WbNY+QSYtkow== +#> ADCCAgoCggIBAKvUq1MOkbPiBfSsSMZsJChPOjHieSt7gWNGJIfh6812SLnu5LDV +#> e/2lz2uu/jF8U0ruenqT8XdgOdZm8x1enJPjWwnNVRag+iS4oynUr5kcFPf2qYr3 +#> mCSA4LIfx0TeDkplUmunmqb9U7GBNLyOFKwYFNX75mBb47pZtUafz0CYx6dIn8Dy +#> yxgVIdy7rk1GcPfMVJwNjSKmXTvHXjr64mN2AhkekqyGLJCJDxw6cZdlGpi+FWdx +#> 7vYhEDanfhAJ/kjgJ0A/Nc5WR2Gj2SqXZLRhPz7i6mb1/zdq+MD+2Xyi1s3xXOP1 +#> s4WwVXkbHzIAR0TpoFfJj8Djosy5PM2E0DiLd36wbGluXnGK8jBLtW6JtHV8equP +#> bZDdRtY/2dfcEvpnqjPY6ISS7cvy7GU7Zud8+MktFMw/vbGOfIH6a9WlEFs0JOeZ +#> tN0RU19UU7Ysl7zfrRmipEYHsyPnAbrvUS9u6nN3F8qtxOOjiFzMwWaT2cSRaZcR +#> V9FS5c+ucfcBNwNIDCJvb+PASwajjm3YKNNO5Y+gmbAj6Odwnf2MBRChgdKRvii7 +#> W9VY1r7PY53o/5DHf7jDzjITaBlFbjP74q4x+JvatYm7aN69+AnP+WE+ByA1vUJf +#> mvkKNQ3t5wQhcYWm5OB5bOk5/oyGZyRK944uENbXi/5Q0mtfR+GatAaJAgMBAAGj +#> VjBUMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFK8Sq6mw5G+3uG6mXiXh +#> TCIF/2VAMB8GA1UdIwQYMBaAFK8Sq6mw5G+3uG6mXiXhTCIF/2VAMA0GCSqGSIb3 +#> DQEBCwUAA4ICAQBuhUUoRi7mlVoZW/aApkPi1kiocRz68Cz4+Uip/4ZcOd1+NKmm +#> HJmFwvWueAZcrfDrWpCWCADB7sh4pWkwqbwQQ7kYmUdRpCGkM6LceYmQk+f1AMQI +#> AGBTUKaTG2gErQl8E+98iQuCyxce48GT/ETXDmEMOvpyYuKxn+EkFrFD5db4i61a +#> DlFmHMH9JnHhxy+E+AN3P3BxQZBbmllUQl7pT1pz00IiSGcrf5imAkM0Rhk4KMcF +#> v1iTcz7nro6oKGDfZk/WNUEbdthnm4we41IJLFKavLOBoBY7X0WvtNQTymGKBUWY +#> y+2CITzgPxcqS1vz3Uf1EIrE70FsBNy4Bzvce88cmOf4Z44ixPNvhCRLwlCP1GuP +#> utnEoSB+PUIapnDdnsYOYeuZOIHDHJ0fdDalHb7alXRLfjw/mO2H2TQ406eveWRo +#> oLF51Uej78lZEn9f0gpgOb9r/WdCK4D+Qrw2ONisaZEx7ILfLSOCLdQbbFcIhf0f +#> n802jHg+TrEQDiJWNQC6qkdR8+GNdIlqx3zlfwH+FgaoaJyqAmgBB/qr5eiYinRT +#> vvVtY5dn8/1VTvI6g/nRzK2pqf1wxKJWTQjGOcThYb8RJ7EBLMvQHtJmbuQGZPHf +#> DDF8ZqYxCffiXjH5UQ3dGnrBkVBKnhFJ7vgrfU81W2sR4uNaRgymPhoF/g== #> -----END CERTIFICATE----- -#> ",""),rs=c(10407,1746256054,887787455,-656355596,-444850843,793160418,489050363))' +#> ",""),rs=c(10407,1366533281,-1114111506,-1933532777,537727212,1698597117,-922383462))' ``` The printed value may be deployed directly on a remote machine. @@ -658,9 +658,11 @@ However it is possible to register custom serialization and unserialization func ``` r serialization( - class = "ArrowTabular", - sfunc = arrow::write_to_raw, - ufunc = function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + list( + arrow::write_to_raw, + function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + ), + class = "ArrowTabular" ) m <- mirai(list(a = head(x), b = "some text"), x = x) @@ -685,9 +687,11 @@ To change registered serialization functions, just call `serialization()` again ``` r serialization( - class = "RPolarsDataFrame", - sfunc = function(x) polars::as_polars_df(x)$to_raw_ipc(), - ufunc = polars::pl$read_ipc + list( + function(x) polars::as_polars_df(x)$to_raw_ipc(), + polars::pl$read_ipc + ), + class = "RPolarsDataFrame" ) x <- polars::as_polars_df(iris) @@ -750,10 +754,10 @@ daemons(4) vec <- c(1, 1, 4, 4, 1, 1, 1, 1) system.time(mirai_map(vec, Sys.sleep)[]) #> user system elapsed -#> 0.007 0.003 4.011 +#> 0.004 0.001 4.008 system.time(parLapply(cl, vec, Sys.sleep)) #> user system elapsed -#> 0.015 0.002 8.014 +#> 0.009 0.005 8.010 ``` `.args` is used to specify further constant arguments to `.f` - the 'mean' and 'sd' in the example below: @@ -763,13 +767,13 @@ with( mirai_map(1:3, rnorm, .args = list(mean = 20, sd = 2))[] ) #> [[1]] -#> [1] 22.96205 +#> [1] 21.02448 #> #> [[2]] -#> [1] 20.17403 21.65934 +#> [1] 19.33405 20.56202 #> #> [[3]] -#> [1] 18.21138 20.59661 23.32587 +#> [1] 19.53189 19.11855 21.52990 ``` Use `...` to further specify objects referenced but not defined in `.f` - the 'do' in the anonymous function below: @@ -782,16 +786,16 @@ ml <- mirai_map( #> Warning in mirai_map(c(a = 1, b = 2, c = 3), function(x) do(x, as.logical(x%%2)), : launching one local daemon as none #> previously set ml -#> < mirai map [3/3] > +#> < mirai map [1/3] > ml[] #> $a -#> [1] "29" +#> [1] "1c" #> #> $b -#> [1] fa 5d +#> [1] d4 c2 #> #> $c -#> [1] "4f99b0" +#> [1] "1ab0a6" ``` Use of `mirai_map()` assumes that `daemons()` have previously been set. If not then one (non-dispatcher) daemon is set to allow the function to proceed. This ensures safe behaviour, but is unlikely to be optimal, so please ensure daemons are set beforehand. diff --git a/vignettes/mirai.Rmd.orig b/vignettes/mirai.Rmd.orig index 42d3f6d60..64b7f3f35 100644 --- a/vignettes/mirai.Rmd.orig +++ b/vignettes/mirai.Rmd.orig @@ -509,9 +509,11 @@ However it is possible to register custom serialization and unserialization func ```{r arrowpass} serialization( - class = "ArrowTabular", - sfunc = arrow::write_to_raw, - ufunc = function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + list( + arrow::write_to_raw, + function(x) arrow::read_ipc_stream(x, as_data_frame = FALSE) + ), + class = "ArrowTabular" ) m <- mirai(list(a = head(x), b = "some text"), x = x) @@ -523,9 +525,11 @@ It can be seen that this time, the arrow table is seamlessly handled in the 'mir To change registered serialization functions, just call `serialization()` again supplying the new functions. As an example, we can switch to using [`polars`](https://pola-rs.github.io/r-polars/), a 'lightning fast' dataframe library written in Rust (requires `polars` >= 0.16.4). ```{r polars} serialization( - class = "RPolarsDataFrame", - sfunc = function(x) polars::as_polars_df(x)$to_raw_ipc(), - ufunc = polars::pl$read_ipc + list( + function(x) polars::as_polars_df(x)$to_raw_ipc(), + polars::pl$read_ipc + ), + class = "RPolarsDataFrame" ) x <- polars::as_polars_df(iris) diff --git a/vignettes/torch.Rmd b/vignettes/torch.Rmd index 8eb8eb5bc..66207a29c 100644 --- a/vignettes/torch.Rmd +++ b/vignettes/torch.Rmd @@ -27,9 +27,8 @@ library(mirai) library(torch) serialization( + fns = list(torch:::torch_serialize, torch::torch_load), class = "torch_tensor", - sfunc = torch:::torch_serialize, - ufunc = torch::torch_load, vec = TRUE ) daemons(1) @@ -77,35 +76,35 @@ The returned model is an object containing many tensor elements. m$data$parameters$conv1.weight #> torch_tensor #> (1,1,.,.) = -#> 0.1984 0.0224 -0.1263 -0.0394 0.1700 -#> -0.0761 0.1168 0.0288 -0.0559 -0.0596 -#> -0.0471 -0.1769 -0.0820 0.0440 -0.1753 -#> -0.0196 0.0442 0.1465 0.1859 -0.0767 -#> -0.1268 0.1959 0.0336 0.1014 0.0487 +#> 0.0036 -0.1690 -0.0054 -0.0737 0.0405 +#> -0.1940 0.0497 -0.0239 -0.0711 -0.0887 +#> 0.0222 -0.0865 0.0335 0.1846 0.0207 +#> -0.1871 -0.1136 -0.0798 -0.0665 0.1499 +#> 0.1467 -0.0570 0.1022 0.0297 -0.0931 #> #> (2,1,.,.) = -#> 0.1073 -0.0402 -0.0262 -0.1252 0.0949 -#> 0.1519 0.1381 0.1774 0.1095 0.1241 -#> -0.1524 0.1307 0.1455 0.0850 -0.1714 -#> 0.0380 -0.1390 0.0885 0.1204 -0.1700 -#> 0.1932 0.1953 0.0262 -0.1436 -0.1753 +#> -0.1554 0.0485 0.1436 -0.0879 0.0130 +#> -0.0645 -0.1229 -0.1542 0.1748 0.1307 +#> -0.1204 0.1478 -0.1953 0.1549 0.1258 +#> 0.0021 -0.1870 -0.1074 0.1557 0.1262 +#> -0.0639 -0.1681 0.1661 0.0434 -0.0977 #> #> (3,1,.,.) = -#> 0.0075 -0.0107 -0.0532 0.1327 -0.0993 -#> 0.1388 0.0941 0.1614 -0.0404 0.0151 -#> -0.0009 0.1057 -0.0208 -0.0372 0.1253 -#> 0.1176 -0.1657 0.0977 -0.1690 0.1061 -#> 0.0036 -0.0701 0.1600 -0.0631 -0.0347 +#> 0.0967 -0.1134 0.0304 -0.1387 0.1591 +#> -0.1895 -0.0770 0.1698 0.0947 -0.1564 +#> -0.1388 0.1359 0.0015 0.0263 -0.0827 +#> -0.0109 0.1353 0.1361 -0.1883 -0.1535 +#> 0.1822 -0.0902 -0.1004 -0.0488 -0.0424 #> #> (4,1,.,.) = -#> 0.0515 -0.1931 -0.1265 -0.1636 -0.0354 -#> 0.1821 -0.0465 0.1698 0.0838 0.1460 -#> -0.1897 0.1488 -0.0529 -0.0432 -0.0576 -#> -0.1919 0.1918 -0.0111 -0.1815 0.1980 -#> -0.0308 0.1593 -0.1766 -0.0209 0.1346 +#> -0.0888 -0.1550 -0.0758 0.0335 0.0973 +#> 0.0543 0.1521 -0.1543 0.0261 0.1008 +#> 0.1672 0.1190 0.0217 -0.0420 0.1000 +#> 0.1382 -0.0775 0.0186 0.1861 -0.0804 +#> 0.0449 0.1972 -0.1447 0.1425 0.1872 #> #> (5,1,.,.) = -#> -0.0533 0.1245 0.1549 -0.0857 0.0338 +#> -0.1344 -0.0403 0.1268 0.0706 0.0973 #> ... [the output was truncated (use n=-1 to disable)] #> [ CPUFloatType{20,1,5,5} ][ requires_grad = TRUE ] ``` diff --git a/vignettes/torch.Rmd.orig b/vignettes/torch.Rmd.orig index ee11da10e..587e1a63d 100644 --- a/vignettes/torch.Rmd.orig +++ b/vignettes/torch.Rmd.orig @@ -32,9 +32,8 @@ library(mirai) library(torch) serialization( + fns = list(torch:::torch_serialize, torch::torch_load), class = "torch_tensor", - sfunc = torch:::torch_serialize, - ufunc = torch::torch_load, vec = TRUE ) daemons(1)