diff --git a/DESCRIPTION b/DESCRIPTION index 05e6693..9055c37 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -13,7 +13,7 @@ Description: In computationally demanding analysis projects, 'clustermq' by Schubert (2019) ), and 'batchtools' by Lang, Bischl, and Surmann (2017). . -Version: 0.0.6.9005 +Version: 0.0.6.9008 License: MIT + file LICENSE URL: https://wlandau.github.io/crew.aws.batch/, https://github.com/wlandau/crew.aws.batch @@ -34,7 +34,7 @@ Depends: R (>= 4.0.0) Imports: cli (>= 3.1.0), - crew (>= 0.9.5.9007), + crew (>= 0.9.5.9012), paws.common (>= 0.7.0), paws.compute, paws.management, @@ -47,6 +47,8 @@ Suggests: markdown (>= 1.1), rmarkdown (>= 2.4), testthat (>= 3.0.0) +Remotes: + wlandau/crew Encoding: UTF-8 Language: en-US Config/testthat/edition: 3 diff --git a/NEWS.md b/NEWS.md index 78431b6..b6cc948 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,10 @@ -# crew.aws.batch 0.0.6.9005 (development) +# crew.aws.batch 0.0.6.9008 (development) * Send both cancellation and termination requests to end jobs. * Fix launcher bug/typo where parameters were supplied to container overrides. * Add a new `all` argument to `terminate()` in the AWS Batch monitor. * Add `r_arguments` to accept command line arguments to R. -* Support `log_resources`. +* Support `options_metrics`. # crew.aws.batch 0.0.6 diff --git a/R/crew_controller_aws_batch.R b/R/crew_controller_aws_batch.R index e5865fc..14a8a7c 100644 --- a/R/crew_controller_aws_batch.R +++ b/R/crew_controller_aws_batch.R @@ -35,7 +35,6 @@ crew_controller_aws_batch <- function( seconds_idle = Inf, seconds_wall = Inf, retry_tasks = TRUE, - log_resources = NULL, tasks_max = Inf, tasks_timers = 0L, reset_globals = TRUE, @@ -45,6 +44,7 @@ crew_controller_aws_batch <- function( launch_max = 5L, processes = NULL, r_arguments = c("--no-save", "--no-restore"), + options_metrics = crew::crew_options_metrics(), aws_batch_config = list(), aws_batch_credentials = list(), aws_batch_endpoint = NULL, @@ -72,8 +72,7 @@ crew_controller_aws_batch <- function( tls_config = tls_config, seconds_interval = seconds_interval, seconds_timeout = seconds_timeout, - retry_tasks = retry_tasks, - log_resources = log_resources + retry_tasks = retry_tasks ) launcher <- crew_launcher_aws_batch( name = name, @@ -92,6 +91,7 @@ crew_controller_aws_batch <- function( tls = tls, processes = processes, r_arguments = r_arguments, + options_metrics = options_metrics, aws_batch_config = aws_batch_config, aws_batch_credentials = aws_batch_credentials, aws_batch_endpoint = aws_batch_endpoint, diff --git a/R/crew_launcher_aws_batch.R b/R/crew_launcher_aws_batch.R index 663bfb0..eaf421b 100644 --- a/R/crew_launcher_aws_batch.R +++ b/R/crew_launcher_aws_batch.R @@ -116,6 +116,7 @@ crew_launcher_aws_batch <- function( tls = crew::crew_tls(mode = "automatic"), processes = NULL, r_arguments = c("--no-save", "--no-restore"), + options_metrics = crew::crew_options_metrics(), aws_batch_config = list(), aws_batch_credentials = list(), aws_batch_endpoint = NULL, @@ -151,6 +152,7 @@ crew_launcher_aws_batch <- function( tls = tls, processes = processes, r_arguments = r_arguments, + options_metrics = options_metrics, aws_batch_config = aws_batch_config, aws_batch_credentials = aws_batch_credentials, aws_batch_endpoint = aws_batch_endpoint, @@ -319,6 +321,7 @@ crew_class_launcher_aws_batch <- R6::R6Class( #' @param tls See [crew_launcher_aws_batch()]. #' @param processes See [crew_launcher_aws_batch()]. #' @param r_arguments See [crew_launcher_aws_batch()]. + #' @param options_metrics See [crew_launcher_aws_batch()]. #' @param aws_batch_config See [crew_launcher_aws_batch()]. #' @param aws_batch_credentials See [crew_launcher_aws_batch()]. #' @param aws_batch_endpoint See [crew_launcher_aws_batch()]. @@ -354,6 +357,7 @@ crew_class_launcher_aws_batch <- R6::R6Class( tls = NULL, processes = NULL, r_arguments = NULL, + options_metrics = NULL, aws_batch_config = NULL, aws_batch_credentials = NULL, aws_batch_endpoint = NULL, @@ -387,7 +391,8 @@ crew_class_launcher_aws_batch <- R6::R6Class( launch_max = launch_max, tls = tls, processes = processes, - r_arguments = r_arguments + r_arguments = r_arguments, + options_metrics = options_metrics ) private$.aws_batch_config <- aws_batch_config private$.aws_batch_credentials <- aws_batch_credentials diff --git a/man/crew_class_launcher_aws_batch.Rd b/man/crew_class_launcher_aws_batch.Rd index e653a1c..3997b49 100644 --- a/man/crew_class_launcher_aws_batch.Rd +++ b/man/crew_class_launcher_aws_batch.Rd @@ -149,6 +149,7 @@ Abstract launcher constructor. tls = NULL, processes = NULL, r_arguments = NULL, + options_metrics = NULL, aws_batch_config = NULL, aws_batch_credentials = NULL, aws_batch_endpoint = NULL, @@ -203,6 +204,8 @@ Abstract launcher constructor. \item{\code{r_arguments}}{See \code{\link[=crew_launcher_aws_batch]{crew_launcher_aws_batch()}}.} +\item{\code{options_metrics}}{See \code{\link[=crew_launcher_aws_batch]{crew_launcher_aws_batch()}}.} + \item{\code{aws_batch_config}}{See \code{\link[=crew_launcher_aws_batch]{crew_launcher_aws_batch()}}.} \item{\code{aws_batch_credentials}}{See \code{\link[=crew_launcher_aws_batch]{crew_launcher_aws_batch()}}.} diff --git a/man/crew_controller_aws_batch.Rd b/man/crew_controller_aws_batch.Rd index 4ac4cb7..981b92e 100644 --- a/man/crew_controller_aws_batch.Rd +++ b/man/crew_controller_aws_batch.Rd @@ -18,7 +18,6 @@ crew_controller_aws_batch( seconds_idle = Inf, seconds_wall = Inf, retry_tasks = TRUE, - log_resources = NULL, tasks_max = Inf, tasks_timers = 0L, reset_globals = TRUE, @@ -28,6 +27,7 @@ crew_controller_aws_batch( launch_max = 5L, processes = NULL, r_arguments = c("--no-save", "--no-restore"), + options_metrics = crew::crew_options_metrics(), aws_batch_config = list(), aws_batch_credentials = list(), aws_batch_endpoint = NULL, @@ -102,37 +102,6 @@ return a \code{mirai} error code (code number 19). Use \code{FALSE} for debugging purposes, e.g. to confirm that a task is causing a worker to run out of memory or crash in some other way.} -\item{log_resources}{Optional character string with a file path to a -text file to log memory consumption. -Set \code{log_resources} to \code{NULL} to avoid writing to a log file. -If you supply a path, then -the \code{log()} method will write memory usage statistics to the file, -and most controller methods will do the same with throttling -so resource consumption is recorded throughout the whole life cycle -of the controller. - -The log file is in comma-separated values -(CSV) format which can be easily read by \code{readr::read_csv()}. -The controller automatically deletes the old log file when it starts -(when \code{controller$start()} is called for the first time, but not -subsequent times). - -The log file has one row per observation of a process, -including the current -R process ("client") and the \code{mirai} dispatcher. If the dispatcher -is not included in the output, it means the dispatcher process -is not running. -Columns include: -* \code{type}: the type of process (client or dispatcher) -* \code{pid}: the process ID. -* \code{status}: The process status (from \code{ps::ps_status()}). -* \code{rss}: resident set size (RSS). RS is the total memory held by -a process, including shared libraries which may also be -in use by other processes. RSS is obtained -from \code{ps::ps_memory_info()} and shown in bytes. -* \code{elapsed}: number of elapsed seconds since the R process was -started (from \code{proc.time()["elapsed"]}).} - \item{tasks_max}{Maximum number of tasks that a worker will do before exiting. See the \code{maxtasks} argument of \code{mirai::daemon()}. \code{crew} does not @@ -178,9 +147,14 @@ using \code{launcher$async$eval()} and expect a \code{mirai} task object as the return value.} \item{r_arguments}{Optional character vector of command line arguments -to pass to R when starting a worker. Example: +to pass to \code{Rscript} (non-Windows) or \code{Rscript.exe} (Windows) +when starting a worker. Example: \code{r_arguments = c("--vanilla", "--max-connections=32")}.} +\item{options_metrics}{Either \code{NULL} to opt out of resource metric logging +for workers, or an object from \code{\link[crew:crew_options_metrics]{crew_options_metrics()}} to enable +and configure resource metric logging for workers.} + \item{aws_batch_config}{Named list, \code{config} argument of \code{paws.compute::batch()} with optional configuration details.} diff --git a/man/crew_launcher_aws_batch.Rd b/man/crew_launcher_aws_batch.Rd index 94cb614..c7e76f8 100644 --- a/man/crew_launcher_aws_batch.Rd +++ b/man/crew_launcher_aws_batch.Rd @@ -21,6 +21,7 @@ crew_launcher_aws_batch( tls = crew::crew_tls(mode = "automatic"), processes = NULL, r_arguments = c("--no-save", "--no-restore"), + options_metrics = crew::crew_options_metrics(), aws_batch_config = list(), aws_batch_credentials = list(), aws_batch_endpoint = NULL, @@ -118,9 +119,14 @@ using \code{launcher$async$eval()} and expect a \code{mirai} task object as the return value.} \item{r_arguments}{Optional character vector of command line arguments -to pass to R when starting a worker. Example: +to pass to \code{Rscript} (non-Windows) or \code{Rscript.exe} (Windows) +when starting a worker. Example: \code{r_arguments = c("--vanilla", "--max-connections=32")}.} +\item{options_metrics}{Either \code{NULL} to opt out of resource metric logging +for workers, or an object from \code{\link[crew:crew_options_metrics]{crew_options_metrics()}} to enable +and configure resource metric logging for workers.} + \item{aws_batch_config}{Named list, \code{config} argument of \code{paws.compute::batch()} with optional configuration details.}