Exit paginator when previous token matches current

paws.common/R/custom_s3.R

@@ -161,22 +161,23 @@ sse_md5_build <- function(request) {
 # encryption key. This handler does both if the MD5 has not been set by
 # the caller.
 sse_md5 <- function(params) {
-  return(.sse_md5(params, 'SSECustomer'))
+  return(.sse_md5(params, "SSECustomer"))
 }
 
 # S3 server-side encryption requires the encryption key to be sent to the
 # server base64 encoded, as well as a base64-encoded MD5 hash of the
 # encryption key. This handler does both if the MD5 has not been set by
 # the caller specifically if the parameter is for the copy-source sse-c key.
 copy_source_sse_md5 <- function(params) {
-  return(.sse_md5(params, 'CopySourceSSECustomer'))
+  return(.sse_md5(params, "CopySourceSSECustomer"))
 }
 
-.sse_md5 <- function(params, sse_member_prefix='SSECustomer') {
-  if (!.needs_s3_sse_customization(params, sse_member_prefix))
+.sse_md5 <- function(params, sse_member_prefix = "SSECustomer") {
+  if (!.needs_s3_sse_customization(params, sse_member_prefix)) {
     return(params)
-  sse_key_member <- paste0(sse_member_prefix, 'Key')
-  sse_md5_member <- paste0(sse_member_prefix, 'KeyMD5')
+  }
+  sse_key_member <- paste0(sse_member_prefix, "Key")
+  sse_md5_member <- paste0(sse_member_prefix, "KeyMD5")
   key_md5_str <- base64enc::base64encode(
     digest::digest(params[[sse_key_member]], serialize = FALSE, raw = TRUE)
   )
@@ -186,9 +187,9 @@ copy_source_sse_md5 <- function(params) {
 }
 
 .needs_s3_sse_customization <- function(params, sse_member_prefix) {
-  return (
-    !is_empty(params[[paste0(sse_member_prefix, 'Key')]]) &
-    is_empty(params[[paste0(sse_member_prefix, 'KeyMD5')]])
+  return(
+    !is_empty(params[[paste0(sse_member_prefix, "Key")]]) &
+      is_empty(params[[paste0(sse_member_prefix, "KeyMD5")]])
   )
 }
 

paws.common/R/paginate.R

@@ -19,6 +19,7 @@
 #' @param MaxItems Limits the maximum number of total returned items returned while paginating.
 #' @param StartingToken Can be used to modify the starting marker or token of a paginator.
 #' This argument if useful for resuming pagination from a previous token or starting pagination at a known position.
+#' @param StopOnSameToken Exist paginator if previous token matches current token.
 #' @param FUN the function to be applied to each response element of \code{operation}.
 #' @param simplify See \link[base:sapply]{base::sapply()}.
 #' @param ... optional arguments to \code{FUN}.
@@ -39,7 +40,8 @@
 paginate <- function(Operation,
                      PageSize = NULL,
                      MaxItems = NULL,
-                     StartingToken = NULL) {
+                     StartingToken = NULL,
+                     StopOnSameToken = FALSE) {
   fn <- substitute(Operation)
   # rebuild fn for do.call
   if (identical(fn[[1]], .do_call)) {
@@ -59,14 +61,26 @@ paginate <- function(Operation,
   while (!identical(fn[[paginator$input_token[[1]]]], character(0))) {
     resp <- eval(fn, envir = parent.frame())
     new_tokens <- get_tokens(resp, paginator$output_token)
+
+    # Exit paginator if previous token matches current token
+    # https://github.com/smithy-lang/smithy-typescript/blob/main/packages/core/src/pagination/createPaginator.ts#L53
+    if (isTRUE(StopOnSameToken)) {
+      previous_token <- unlist(fn[[paginator$input_token]], use.names = F)
+      if (identical(previous_token, unlist(new_tokens, use.names = F))) {
+        break
+      }
+    }
+
+    # Update tokens
     for (i in seq_along(new_tokens)) {
       fn[[paginator$input_token[[i]]]] <- new_tokens[[i]]
     }
+
     result[[length(result) + 1]] <- resp
 
     # exit if no more results
-    if (!is.null(paginator$more_results)) {
-      if (isFALSE(resp[[paginator$more_results]])) {
+    if (!is.null(paginator[["more_results"]])) {
+      if (isFALSE(resp[[paginator[["more_results"]]]])) {
         break
       }
     }
@@ -87,7 +101,8 @@ paginate_lapply <- function(Operation,
                             ...,
                             PageSize = NULL,
                             MaxItems = NULL,
-                            StartingToken = NULL) {
+                            StartingToken = NULL,
+                            StopOnSameToken = FALSE) {
   FUN <- match.fun(FUN)
   fn <- substitute(Operation)
 
@@ -106,7 +121,8 @@ paginate_lapply <- function(Operation,
     paginator = fn_update$paginator,
     FUN = FUN,
     ...,
-    MaxItems = MaxItems
+    MaxItems = MaxItems,
+    StopOnSameToken = StopOnSameToken
   )
   return(result)
 }
@@ -119,7 +135,8 @@ paginate_sapply <- function(Operation,
                             simplify = TRUE,
                             PageSize = NULL,
                             MaxItems = NULL,
-                            StartingToken = NULL) {
+                            StartingToken = NULL,
+                            StopOnSameToken = FALSE) {
   FUN <- match.fun(FUN)
   fn <- substitute(Operation)
 
@@ -138,7 +155,8 @@ paginate_sapply <- function(Operation,
     paginator = fn_update$paginator,
     FUN = FUN,
     ...,
-    MaxItems = MaxItems
+    MaxItems = MaxItems,
+    StopOnSameToken = StopOnSameToken
   )
 
   if (!isFALSE(simplify)) {
@@ -234,21 +252,34 @@ paginate_xapply <- function(
     paginator,
     FUN,
     ...,
-    MaxItems = NULL) {
+    MaxItems = NULL,
+    StopOnSameToken = FALSE) {
   primary_result_key <- paginator$result_key[[1]]
   no_items <- 0
   result <- list()
   while (!identical(fn[[paginator$input_token[[1]]]], character(0))) {
     resp <- eval(fn, envir = parent.frame(n = 2))
     new_tokens <- get_tokens(resp, paginator$output_token)
+
+    # Exit paginator if previous token matches current token
+    # https://github.com/smithy-lang/smithy-typescript/blob/main/packages/core/src/pagination/createPaginator.ts#L53
+    if (isTRUE(StopOnSameToken)) {
+      previous_token <- unlist(fn[[paginator$input_token]], use.names = F)
+      if (identical(previous_token, unlist(new_tokens, use.names = F))) {
+        break
+      }
+    }
+
+    # Update tokens
     for (i in seq_along(new_tokens)) {
       fn[[paginator$input_token[[i]]]] <- new_tokens[[i]]
     }
+
     result[[length(result) + 1]] <- FUN(resp, ...)
 
     # exit if no more results
-    if (!is.null(paginator$more_results)) {
-      if (isFALSE(resp[[paginator$more_results]])) {
+    if (!is.null(paginator[["more_results"]])) {
+      if (isFALSE(resp[[paginator[["more_results"]]]])) {
         break
       }
     }

paws.common/tests/testthat/test_paginate.R

@@ -164,6 +164,57 @@ test_that("check paginate", {
   expect_equal(actual, expected)
 })
 
+test_that("check paginate stop on same token", {
+  dummy_internal <- function(paginator) {
+    paginator
+  }
+  dummy_op <- function(x, NextToken = NULL, MaxKey = NULL) {
+    op <- dummy_internal(paginator = list(
+      input_token = "NextToken",
+      output_token = "NextToken",
+      limit_key = "MaxKey",
+      result_key = "Contents"
+    ))
+    list(NextToken = NextToken, MaxKey = MaxKey)
+  }
+  mock_substitute <- mock2(substitute(dummy_op(x = "hi")))
+
+  mock_paginate_update_fn <- mock2(
+    list(
+      fn = substitute(dummy_op(x = "hi")),
+      paginator = list(
+        input_token = "NextToken",
+        output_token = "NextToken",
+        limit_key = "MaxKey",
+        result_key = "Contents"
+      )
+    )
+  )
+  mock_eval <- mock2(
+    list(Contents = list("foo"), NextToken = "token1"),
+    list(Contents = list("bar"), NextToken = "token2"),
+    list(Contents = list(), NextToken = "token2")
+  )
+
+  expected <- list(
+    list(Contents = list("foo"), NextToken = "token1"),
+    list(Contents = list("bar"), NextToken = "token2")
+  )
+
+  mockery::stub(paginate, "substitute", mock_substitute)
+  mockery::stub(paginate, "paginate_update_fn", mock_paginate_update_fn)
+  mockery::stub(paginate, "eval", mock_eval)
+
+  actual <- paginate("dummy", StopOnSameToken = TRUE)
+  actual_args <- mockery::mock_args(mock_eval)
+  expect_equal(lapply(actual_args, function(x) as.list(x[[1]][-1])), list(
+    list(x = "hi"),
+    list(x = "hi", NextToken = "token1"),
+    list(x = "hi", NextToken = "token2")
+  ))
+  expect_equal(actual, expected)
+})
+
 ########################################################################
 # paginate_xapply
 ########################################################################
@@ -267,6 +318,55 @@ test_that("check paginate_xapply restrict MaxItems", {
   expect_equal(actual, expected)
 })
 
+test_that("check paginate stop on same token", {
+  dummy_internal <- function(paginator) {
+    paginator
+  }
+  dummy_op <- function(x, NextToken = NULL, MaxKey = NULL) {
+    op <- dummy_internal(paginator = list(
+      input_token = "NextToken",
+      output_token = "NextToken",
+      limit_key = "MaxKey",
+      result_key = "Contents"
+    ))
+    list(NextToken = NextToken, MaxKey = MaxKey)
+  }
+
+  mock_eval <- mock2(
+    list(Contents = list("foo"), NextToken = "token1"),
+    list(Contents = list("bar"), NextToken = "token2"),
+    list(Contents = list(), NextToken = "token2")
+  )
+
+  expected <- list(
+    list(Contents = list("foo"), NextToken = "token1"),
+    list(Contents = list("bar"), NextToken = "token2")
+  )
+
+  mockery::stub(paginate_xapply, "eval", mock_eval)
+
+  actual <- paginate_xapply(
+    substitute(dummy_op(x = "hi")),
+    paginator = list(
+      input_token = "NextToken",
+      output_token = "NextToken",
+      limit_key = "MaxKey",
+      result_key = "Contents"
+    ),
+    FUN = function(resp) {
+      resp
+    },
+    MaxItems = NULL,
+    StopOnSameToken = TRUE
+  )
+  actual_args <- mockery::mock_args(mock_eval)
+  expect_equal(lapply(actual_args, function(x) as.list(x[[1]][-1])), list(
+    list(x = "hi"),
+    list(x = "hi", NextToken = "token1"),
+    list(x = "hi", NextToken = "token2")
+  ))
+  expect_equal(actual, expected)
+})
 
 ########################################################################
 # paginate_lapply