better plumbing

NAMESPACE

@@ -41,7 +41,6 @@ export(retrieve_credential_local)
 export(retrieve_credential_mssql)
 export(sanitize_token)
 export(validate_field_names)
-export(validate_field_names_collapsed)
 export(validate_for_write)
 export(validate_no_logical)
 importFrom(magrittr,"%>%")

NEWS.md

@@ -15,12 +15,39 @@ These features are not yet on CRAN.  Install with `remotes::install_github("Ouhs
 
 ### Possibly Breaking Change
 
-* `redcap_read()`, `redcap_read_oneshot()`, `redcap_dag_read()`, `redcap_log_read()` and `redcap_report()` return a [tibble](https://tibble.tidyverse.org/) instead of a [data.frame](https://stat.ethz.ch/R-manual/R-devel/library/base/html/data.frame.html).  (#415)
+The could possibly break existing code --but it's very unlikely.  We don't like risking it, but feel these changes will(directly and indirectly) considerably improve the package.
+
+* `redcap_read()`, `redcap_read_oneshot()`, `redcap_dag_read()`, `redcap_log_read()`, and `redcap_report()` return a [tibble](https://tibble.tidyverse.org/) instead of a [data.frame](https://stat.ethz.ch/R-manual/R-devel/library/base/html/data.frame.html).  (#415)
 
     This should affect client code only if you expect a call like `ds[, 3]` to return a vector instead of a single-column data.frame/tibble.  One solution is to upcast the tibble to a data.frame (with something like `as.data.frame()`).  We recommend using an approach that works for both data.frames and tibbles, such as `ds[[3]]` or `dplyr::pull(ds, "gender")`.
 
     For more information, read the short chapter in [*R for Data Science*](https://r4ds.had.co.nz/tibbles.html).
 
+* The `*_collapsed` parameters are deprecated.  When your want to limit on records/fields/forms/events, pass the vector of characters, not the scalar character separated by commas (which I think everyone does already).  In other words use `c("demographics", "blood_pressure")` instead of `"demographics,blood_pressure"`.
+
+    Here are the relationships between the four pairs of variables:
+
+    ```r
+    records_collapsed   <- collapse_vector(records  , records_collapsed)
+    fields_collapsed    <- collapse_vector(fields   , fields_collapsed)
+    forms_collapsed     <- collapse_vector(forms    , forms_collapsed)
+    events_collapsed    <- collapse_vector(events   , events_collapsed)
+    ```
+
+    If someone is using the *_collapsed parameter, they can programmatically convert it to a vector like:
+
+    ```r
+    field_names <- trimws(unlist(strsplit(field_names_collapsed, ",")))
+    ```
+
+* `redcap_read()` will automatically include the "plumbing" variables, even if they're not included the list of requested fields & forms.  (#442).  Specifically:
+
+    * `record_id` (or it's customized name) will always be returned
+    * `redcap_event_name` will be returned for longitudinal projects
+    * `redcap_repeat_instrument` and `redcap_repeat_instance` will be returned for projects with repeating instruments
+
+This will help extract forms from longitudinal & repeating projects.
+
 ### New Features
 
 * New `redcap_metadata_coltypes()` function.  Inspects the fields types and validation text of each field to generate a suggested `readr::col_types` object that reflects the project's current data dictionary.  The object then can be passed to the `col_types` parameter of `redcap_read()` or `redcap_read_oneshot()`.  (Suggested and discussed with @pbchase, @nutterb, @skadauke, & others, #405 & #294)

R/redcap-event-instruments.R

@@ -83,7 +83,7 @@ redcap_event_instruments <- function(
     token     = token,
     content   = "formEventMapping",
     format    = "csv",
-    "arms[0]" = collapse_vector(arms, NULL)
+    "arms[0]" = collapse_vector(arms)
   )
 
   col_types <- readr::cols(

R/redcap-metadata-coltypes.R

@@ -106,7 +106,7 @@
 #' the project manager selected the "integer" validation option, all those
 #' "abcd" values remain untouched.
 #'
-#' This is one reason `redcap_metadata_coltypes` prints it suggestions to the console.
+#' This is one reason `redcap_metadata_coltypes()` prints it suggestions to the console.
 #' It allows the developer to adjust the specifications to match the values
 #' returned by the API.  The the "abcd" scenario, consider (a) changing the type
 #' from `col_integer` to `col_character`, (b) excluding the trash values,
@@ -237,8 +237,9 @@ redcap_metadata_internal <- function(
   d_proj <- REDCapR::redcap_project_info_read(redcap_uri, token, verbose = verbose, handle_httr = handle_httr)$data
 
   # Determine status of autonumbering, instrument complete status, and decimal mark
-  .record_field         <- d_var$original_field_name[1]
+  .record_field         <- d_var$original_field_name[1] # The first field should always be the "record" identifier.
   .autonumber           <- d_proj$record_autonumbering_enabled[1]
+  .plumbing_possibles    <- c(.record_field, "redcap_event_name", "redcap_repeat_instrument", "redcap_repeat_instance")
   decimal_period        <- (locale$decimal_mark == ".")
   decimal_comma         <- (locale$decimal_mark == ",")
 
@@ -344,8 +345,12 @@ redcap_metadata_internal <- function(
     dplyr::ungroup() %>%
     dplyr::arrange(.data$form_order,  .data$field_order_within_form) %>%
     dplyr::select(-.data$form_order, -.data$field_order_within_form) %>%
-    tibble::add_row(d_again, .after = 1)
+    tibble::add_row(d_again, .after = 1) %>%
+    dplyr::mutate(
+      plumbing = (.data$field_name %in% .plumbing_possibles)
+    )
 
+  # The types of variables that are in metadata, but NOT variables:
   # setdiff(d_meta$field_name_base, d_var$original_field_name)
   # [1] "signature"   "file_upload" "descriptive"
 
@@ -451,9 +456,12 @@ redcap_metadata_internal <- function(
       # .data$padding1,
       # .data$padding2,
       .data$aligned,
-      .data$field_name_base
+      .data$field_name_base,
+      .data$plumbing
     )
 
+  .plumbing_variables <- intersect(d$field_name, .plumbing_possibles)
+
   decimal_period_any <- any(d_meta$vt %in% c("number", "number_1dp", "number_2dp", "number_3dp", "number_4dp" ))
   decimal_comma_any  <- any(d_meta$vt %in% c("number_comma_decimal", "number_1dp_comma_decimal", "number_2dp_comma_decimal", "number_3dp_comma_decimal", "number_4dp_comma_decimal"))
 
@@ -472,7 +480,21 @@ redcap_metadata_internal <- function(
 
   list(
     d_variable      = d,
+    success         = TRUE,
     longitudinal    = d_proj$is_longitudinal[1],
-    repeating       = d_proj$has_repeating_instruments_or_events[1]
+    repeating       = d_proj$has_repeating_instruments_or_events[1],
+    record_id_name  = .record_field,
+    plumbing_variables = .plumbing_variables
   )
 }
+
+# uri   <- "https://bbmc.ouhsc.edu/redcap/api/"
+#
+# A simple project (pid 153)
+# REDCapR:::redcap_metadata_internal(uri, "9A81268476645C4E5F03428B8AC3AA7B")$d_variable
+#
+# A longitudinal project (pid 212)
+# REDCapR:::redcap_metadata_internal(uri, "0434F0E9CF53ED0587847AB6E51DE762")$d_variable
+#
+# # A repeating measures (pid 3181)
+# REDCapR:::redcap_metadata_internal(uri, "22C3FF1C8B08899FB6F86D91D874A159")$d_variable

R/redcap-metadata-read.R

@@ -15,12 +15,8 @@
 #' project.  Required.
 #' @param forms An array, where each element corresponds to the REDCap form
 #' of the desired fields.  Optional.
-#' @param forms_collapsed A single string, where the desired forms are
-#' separated by commas.  Optional.
 #' @param fields An array, where each element corresponds to a desired project
 #' field.  Optional.
-#' @param fields_collapsed A single string, where the desired field names are
-#' separated by commas.  Optional.
 #' @param verbose A boolean value indicating if `message`s should be printed
 #' to the R console during the operation.  The verbose output might contain
 #' sensitive information (*e.g.* PHI), so turn this off if the output might
@@ -64,8 +60,18 @@
 #' @examples
 #' \dontrun{
 #' uri   <- "https://bbmc.ouhsc.edu/redcap/api/"
+#'
+#' # A simple project (pid 153)
 #' token <- "9A81268476645C4E5F03428B8AC3AA7B"
 #' REDCapR::redcap_metadata_read(redcap_uri=uri, token=token)
+#'
+#' # A longitudinal project (pid 212)
+#' token <- "0434F0E9CF53ED0587847AB6E51DE762"
+#' REDCapR::redcap_metadata_read(redcap_uri=uri, token=token)
+#'
+#' # A repeating measures (pid 3181)
+#' token <- "22C3FF1C8B08899FB6F86D91D874A159"
+#' REDCapR::redcap_metadata_read(redcap_uri=uri, token=token)
 #' }
 
 #' @importFrom magrittr %>%
@@ -74,9 +80,7 @@ redcap_metadata_read <- function(
   redcap_uri,
   token,
   forms             = NULL,
-  forms_collapsed   = "",
   fields            = NULL,
-  fields_collapsed  = "",
   verbose           = TRUE,
   config_options    = NULL,
   handle_httr       = NULL
@@ -88,13 +92,10 @@ redcap_metadata_read <- function(
   validate_field_names(fields, stop_on_error = TRUE)
 
   token               <- sanitize_token(token)
-  fields_collapsed    <- collapse_vector(fields   , fields_collapsed)
-  forms_collapsed     <- collapse_vector(forms    , forms_collapsed)
+  fields_collapsed    <- collapse_vector(fields)
+  forms_collapsed     <- collapse_vector(forms)
   verbose             <- verbose_prepare(verbose)
 
-  if (1L <= nchar(fields_collapsed))
-    validate_field_names_collapsed(fields_collapsed, stop_on_error = TRUE)
-
   post_body <- list(
     token    = token,
     content  = "metadata",

R/redcap-project.R

@@ -59,13 +59,9 @@ redcap_project <- setRefClass(
       batch_size                  = 100L,
       interbatch_delay            = 0,
       records                     = NULL,
-      records_collapsed           = "",
       fields                      = NULL,
-      fields_collapsed            = "",
       forms                       = NULL,
-      forms_collapsed             = "",
       events                      = NULL,
-      events_collapsed            = "",
       raw_or_label                = "raw",
       raw_or_label_headers        = "raw",
       export_checkbox_label       = FALSE,
@@ -89,13 +85,9 @@ redcap_project <- setRefClass(
         redcap_uri                    = redcap_uri,
         token                         = token,
         records                       = records,
-        records_collapsed             = records_collapsed,
         fields                        = fields,
-        fields_collapsed              = fields_collapsed,
         forms                         = forms,
-        forms_collapsed               = forms_collapsed,
         events                        = events,
-        events_collapsed              = events_collapsed,
         raw_or_label                  = raw_or_label,
         raw_or_label_headers          = raw_or_label_headers,
         export_checkbox_label         = export_checkbox_label,

R/redcap-read-eav-oneshot.R

@@ -14,20 +14,12 @@
 #' project.  Required.
 #' @param records An array, where each element corresponds to the ID of a
 #' desired record.  Optional.
-#' @param records_collapsed A single string, where the desired ID values
-#' are separated by commas.  Optional.
 #' @param fields An array, where each element corresponds to a desired project
 #' field.  Optional.
-#' @param fields_collapsed A single string, where the desired field names are
-#' separated by commas.  Optional.
 #' @param forms An array, where each element corresponds to a desired project
 #' form.  Optional.
-#' @param forms_collapsed A single string, where the desired form names are
-#' separated by commas.  Optional.
 #' @param events An array, where each element corresponds to a desired project
 #' event.  Optional.
-#' @param events_collapsed A single string, where the desired event names are
-#' separated by commas.  Optional.
 # @param raw_or_label A string (either `'raw'` or `'label'`) that specifies
 # whether to export the raw coded values or the labels for the options of
 # multiple choice fields.  Default is `'raw'`.
@@ -183,13 +175,9 @@ redcap_read_eav_oneshot <- function(
     redcap_uri,
     token,
     records                       = NULL,
-    records_collapsed             = "",
     fields                        = NULL,
-    fields_collapsed              = "",
     forms                         = NULL,
-    forms_collapsed               = "",
     events                        = NULL,
-    events_collapsed              = "",
     # raw_or_label                  = "raw",
     # raw_or_label_headers          = "raw",
     # export_checkbox_label         = FALSE,
@@ -214,13 +202,9 @@ redcap_read_eav_oneshot <- function(
   checkmate::assert_character(redcap_uri                , any.missing=FALSE, len=1, pattern="^.{1,}$")
   checkmate::assert_character(token                     , any.missing=FALSE, len=1, pattern="^.{1,}$")
   checkmate::assert_atomic(records                      , any.missing=TRUE , min.len=0)
-  checkmate::assert_character(records_collapsed         , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE)
   checkmate::assert_character(fields                    , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE)
-  checkmate::assert_character(fields_collapsed          , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE)
   checkmate::assert_character(forms                     , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE)
-  checkmate::assert_character(forms_collapsed           , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE)
   checkmate::assert_character(events                    , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE)
-  checkmate::assert_character(events_collapsed          , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE)
 #   checkmate::assert_character(raw_or_label              , any.missing=FALSE, len=1)
 #   checkmate::assert_subset(   raw_or_label              , c("raw", "label"))
 #   checkmate::assert_character(raw_or_label_headers      , any.missing=FALSE, len=1)
@@ -246,18 +230,15 @@ redcap_read_eav_oneshot <- function(
   validate_field_names(fields, stop_on_error = TRUE)
 
   token               <- sanitize_token(token)
-  records_collapsed   <- collapse_vector(records  , records_collapsed)
-  fields_collapsed    <- collapse_vector(fields   , fields_collapsed)
-  forms_collapsed     <- collapse_vector(forms    , forms_collapsed)
-  events_collapsed    <- collapse_vector(events   , events_collapsed)
+  records_collapsed   <- collapse_vector(records)
+  fields_collapsed    <- collapse_vector(fields)
+  forms_collapsed     <- collapse_vector(forms)
+  events_collapsed    <- collapse_vector(events)
   filter_logic        <- filter_logic_prepare(filter_logic)
   datetime_range_begin<- dplyr::coalesce(strftime(datetime_range_begin, "%Y-%m-%d %H:%M:%S"), "")
   datetime_range_end  <- dplyr::coalesce(strftime(datetime_range_end  , "%Y-%m-%d %H:%M:%S"), "")
   verbose             <- verbose_prepare(verbose)
 
-  if (1L <= nchar(fields_collapsed) )
-    validate_field_names_collapsed(fields_collapsed, stop_on_error = TRUE)
-
   post_body <- list(
     token                   = token,
     content                 = "record",

R/redcap-read-oneshot-eav.R

@@ -14,20 +14,12 @@
 #' project.  Required.
 #' @param records An array, where each element corresponds to the ID of a
 #' desired record.  Optional.
-#' @param records_collapsed A single string, where the desired ID values
-#' are separated by commas.  Optional.
 #' @param fields An array, where each element corresponds to a desired
 #' project field.  Optional.
-#' @param fields_collapsed A single string, where the desired field names
-#' are separated by commas.  Optional.
 #' @param forms An array, where each element corresponds to a desired project
 #' field.  Optional.
-#' @param forms_collapsed A single string, where the desired form names are
-#' separated by commas.  Optional.
 #' @param events An array, where each element corresponds to a desired project
 #' event.  Optional.
-#' @param events_collapsed A single string, where the desired event names are
-#' separated by commas.  Optional.
 #' @param raw_or_label A string (either `'raw'` or `'label'` that specifies
 #' whether to export the raw coded values or the labels for the options of
 #' multiple choice fields.  Default is `'raw'`.
@@ -131,15 +123,15 @@
 #'
 #' # Return only records with IDs of 1 and 3
 #' desired_records_v1 <- c(1, 3)
-#' ds_some_rows_v1    <- REDCapR::redcap_read_oneshot_eav(
+#' ds_some_rows_v1    <- REDCapR:::redcap_read_oneshot_eav(
 #'    redcap_uri = uri,
 #'    token      = token,
 #'    records    = desired_records_v1
 #' )$data
 #'
 #' # Return only the fields record_id, name_first, and age
 #' desired_fields_v1 <- c("record_id", "name_first", "age")
-#' ds_some_fields_v1 <- redcap_read_oneshot_eav(
+#' ds_some_fields_v1 <- REDCapR:::redcap_read_oneshot_eav(
 #'    redcap_uri = uri,
 #'    token      = token,
 #'    fields     = desired_fields_v1
@@ -154,13 +146,9 @@ redcap_read_oneshot_eav <- function(
   redcap_uri,
   token,
   records                       = NULL,
-  records_collapsed             = "",
   fields                        = NULL,
-  fields_collapsed              = "",
   forms                         = NULL,
-  forms_collapsed               = "",
   events                        = NULL,
-  events_collapsed              = "",
   raw_or_label                  = "raw",
   raw_or_label_headers          = "raw",
   # placeholder: exportCheckboxLabel
@@ -184,13 +172,9 @@ redcap_read_oneshot_eav <- function(
   checkmate::assert_character(redcap_uri                , any.missing=FALSE, len=1, pattern="^.{1,}$")
   checkmate::assert_character(token                     , any.missing=FALSE, len=1, pattern="^.{1,}$")
   checkmate::assert_atomic(records                      , any.missing=TRUE , min.len=0)
-  checkmate::assert_character(records_collapsed         , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE )
   checkmate::assert_character(fields                    , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE )
-  checkmate::assert_character(fields_collapsed          , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE )
   checkmate::assert_character(forms                     , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE )
-  checkmate::assert_character(forms_collapsed           , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE )
   checkmate::assert_character(events                    , any.missing=TRUE , min.len=1, pattern="^.{1,}$", null.ok=TRUE )
-  checkmate::assert_character(events_collapsed          , any.missing=TRUE , len=1, pattern="^.{0,}$", null.ok=TRUE)
   checkmate::assert_character(raw_or_label              , any.missing=FALSE, len=1)
   checkmate::assert_subset(   raw_or_label              , c("raw", "label"))
   checkmate::assert_character(raw_or_label_headers      , any.missing=FALSE, len=1)
@@ -214,19 +198,16 @@ redcap_read_oneshot_eav <- function(
   validate_field_names(fields, stop_on_error = TRUE)
 
   token                     <- sanitize_token(token)
-  records_collapsed         <- collapse_vector(records  , records_collapsed)
-  fields_collapsed          <- collapse_vector(fields   , fields_collapsed)
-  forms_collapsed           <- collapse_vector(forms    , forms_collapsed)
-  events_collapsed          <- collapse_vector(events   , events_collapsed)
+  records_collapsed         <- collapse_vector(records)
+  fields_collapsed          <- collapse_vector(fields)
+  forms_collapsed           <- collapse_vector(forms)
+  events_collapsed          <- collapse_vector(events)
   export_data_access_groups <- ifelse(export_data_access_groups, "true", "false")
   filter_logic              <- filter_logic_prepare(filter_logic)
   datetime_range_begin      <- dplyr::coalesce(strftime(datetime_range_begin, "%Y-%m-%d %H:%M:%S"), "")
   datetime_range_end        <- dplyr::coalesce(strftime(datetime_range_end  , "%Y-%m-%d %H:%M:%S"), "")
   verbose                   <- verbose_prepare(verbose)
 
-  if (1L <= nchar(fields_collapsed) )
-    validate_field_names_collapsed(fields_collapsed, stop_on_error = TRUE)
-
   post_body <- list(
     token                        = token,
     content                      = "record",
@@ -268,7 +249,7 @@ redcap_read_oneshot_eav <- function(
     REDCapR::redcap_metadata_read(
       redcap_uri,
       token,
-      forms_collapsed = forms_collapsed,
+      forms           = forms,
       handle_httr     = handle_httr
     )$data