diff --git a/DESCRIPTION b/DESCRIPTION index 494df321..85f91ad5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: rstac Title: Client Library for SpatioTemporal Asset Catalog -Version: 1.0.0 +Version: 1.0.1 Authors@R: c(person("Rolf", "Simoes", email = "rolfsimoes@gmail.com", @@ -38,7 +38,14 @@ Imports: Suggests: lifecycle, testthat, - knitr + knitr, + tmap, + leaflet, + stars, + slider, + ggplot2, + purrr, + dplyr Collate: 'cql2-expr-funs.R' 'cql2-types.R' @@ -54,6 +61,7 @@ Collate: 'assets-funs.R' 'check-utils.R' 'conformance-query.R' + 'collections-funs.R' 'collections-query.R' 'deprec-funs.R' 'doc-funs.R' @@ -80,3 +88,4 @@ Collate: 'rstac.R' 'rstac-funs.R' Roxygen: list(markdown = TRUE) +VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index 5400e132..4a713392 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -174,6 +174,10 @@ export(assets_rename) export(assets_select) export(assets_url) export(collections) +export(collections_fetch) +export(collections_length) +export(collections_matched) +export(collections_next) export(conformance) export(cql2_bbox_as_geojson) export(cql2_date) diff --git a/NEWS.md b/NEWS.md index 0d1e8a1f..d2e5ae4a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +# rstac (development version) + # rstac 1.0.0 (Released 2024-02-14) * Add support to static catalogs; diff --git a/R/assets-funs.R b/R/assets-funs.R index 47df0203..35e10494 100644 --- a/R/assets-funs.R +++ b/R/assets-funs.R @@ -43,6 +43,9 @@ #' @param progress a `logical` indicating if a progress bar must be #' shown or not. Defaults to `TRUE`. #' +#' @param use_gdal a `logical` indicating if the file should be downloaded +#' by GDAL instead httr package. +#' #' @param download_fn a `function` to handle download of assets for #' each item to be downloaded. Using this function, you can change the #' hrefs for each asset, as well as the way download is done. @@ -182,6 +185,7 @@ assets_download <- function(items, asset_names = NULL, output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, download_fn = NULL) { # check output dir if (!dir.exists(output_dir)) @@ -197,6 +201,7 @@ assets_download.doc_item <- function(items, asset_names = NULL, output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, create_json = FALSE, download_fn = NULL) { if (!is.null(asset_names)) { @@ -209,7 +214,7 @@ assets_download.doc_item <- function(items, } items$assets <- lapply( items$assets, asset_download, output_dir = output_dir, - overwrite = overwrite, ..., download_fn = download_fn + overwrite = overwrite, use_gdal = use_gdal, download_fn = download_fn, ... ) if (create_json) { file <- "item.json" @@ -228,6 +233,7 @@ assets_download.doc_items <- function(items, asset_names = NULL, output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, download_fn = NULL, create_json = TRUE, items_max = Inf, @@ -249,7 +255,7 @@ assets_download.doc_items <- function(items, items$features[[i]] <- assets_download( items = items$features[[i]], asset_names = asset_names, output_dir = output_dir, overwrite = overwrite, - create_json = FALSE, download_fn = download_fn, ... + use_gdal = use_gdal, create_json = FALSE, download_fn = download_fn, ... ) } if (create_json) diff --git a/R/assets-utils.R b/R/assets-utils.R index ade3f611..6fa36619 100644 --- a/R/assets-utils.R +++ b/R/assets-utils.R @@ -56,19 +56,36 @@ select_exec <- function(key, asset, select_fn) { asset_download <- function(asset, output_dir, overwrite, ..., + use_gdal = FALSE, download_fn = NULL) { if (!is.null(download_fn)) return(download_fn(asset)) # create a full path name - path <- url_get_path(asset$href) - out_file <- path_normalize(output_dir, path) - dir_create(out_file) - make_get_request( - url = asset$href, - httr::write_disk(path = out_file, overwrite = overwrite), - ..., - error_msg = "Error while downloading" - ) - asset$href <- path + out_file <- path_normalize(output_dir, url_get_path(asset$href)) + out_dir <- dirname(out_file) + if (!dir.exists(out_dir)) + dir.create(out_dir, recursive = TRUE) + stopifnot(dir.exists(out_dir)) + if (use_gdal) { + if (file.exists(out_file) && !overwrite) + .error("File already exists. Use `overwrite=TRUE`.") + if (file.exists(out_file)) + unlink(out_file) + sf::gdal_utils( + util = "translate", + source = gdalvsi_append(asset$href), + destination = out_file, ... + ) + if (!file.exists(out_file)) { + .error("Download failed. File: '%s'.", asset$href) + } + } else { + make_get_request( + url = asset$href, + httr::write_disk(path = out_file, overwrite = overwrite), + error_msg = "Error while downloading", ... + ) + } + asset$href <- out_file asset } diff --git a/R/check-utils.R b/R/check-utils.R index f2200e04..4cdbb0c0 100644 --- a/R/check-utils.R +++ b/R/check-utils.R @@ -74,6 +74,14 @@ check_collection <- function(collection) { collection } +check_collections <- function(collections) { + if (!is.list(collections) || is.null(names(collections))) + .error("Invalid doc_collections object.") + if (!"links" %in% names(collections)) + .error("Invalid doc_collections object. Expecting `links` key.") + collections +} + check_character <- function(x, msg, ...) { if (!is.character(x)) .error(msg, ...) diff --git a/R/collections-funs.R b/R/collections-funs.R new file mode 100644 index 00000000..0bcaeb3a --- /dev/null +++ b/R/collections-funs.R @@ -0,0 +1,161 @@ +#' @title Collections functions +#' +#' @description +#' These functions provide support to work with +#' `doc_collections`objects. +#' +#' \itemize{ +#' \item `collections_length()`: `r lifecycle::badge('experimental')` +#' shows how many items there are in the `doc_items` object. +#' +#' \item `collections_matched()`: `r lifecycle::badge('experimental')` +#' shows how many items matched the search criteria. +#' +#' \item `collections_fetch()`: `r lifecycle::badge('experimental')` +#' request all STAC Items through pagination. +#' +#' \item `collections_next()`: `r lifecycle::badge('experimental')` +#' fetches a new page from STAC service. +#' +#' } +#' +#' @param collections a `doc_collections` object. +#' +#' @param matched_field a `character` vector with the path +#' where is the number of collections returned. +#' +#' @param progress a `logical` indicating if a progress bar must be +#' shown or not. Defaults to `TRUE`. +#' +#' @param ... additional arguments. See details. +#' +#' @details +#' Ellipsis argument (`...`) appears in different items functions and +#' has distinct purposes: +#' +#' \itemize{ +#' \item `collections_fetch()` and `collections_next()`: ellipsis is used to +#' pass additional `httr` options to [GET][httr::GET] method, such as +#' [add_headers][httr::add_headers] or [set_cookies][httr::set_cookies]. +#' +#' } +#' +#' @return +#' +#' \itemize{ +#' \item `collections_length()`: an `integer` value. +#' +#' \item `collections_matched()`: returns an `integer` value if the STAC web +#' server does support this extension. Otherwise returns `NULL`. +#' +#' \item `collections_fetch()`: a `doc_items` with all matched items. +#' +#' \item `collections_next()`: fetches a new page from STAC service. +#' +#' } +#' +#' @examples +#' \dontrun{ +#' # doc_items object +#' stac("https://cmr.earthdata.nasa.gov/stac/LPCLOUD") |> +#' collections() |> +#' get_request() |> +#' collections_fetch() +#' } +#' +#' @name collections_functions +NULL + + + +#' @rdname collections_functions +#' +#' @export +collections_next <- function(collections, ...) { + check_collection(collections) + # get url of the next page + rel <- NULL + next_link <- links(collections, rel == "next") + if (length(next_link) == 0) + .error("Cannot get next link URL.", class = "next_error") + next_link <- next_link[[1]] + res <- make_get_request( + url = next_link$href, + headers = next_link$headers, + ..., + error_msg = "Error while requesting next page" + ) + content <- content_response_json(res) + # return items + doc_collections(content) +} + +#' @rdname collections_functions +#' +#' @export +collections_matched <- function(collections, matched_field) { + check_collections(collections) + matched <- NULL + if (is.character(matched_field) && matched_field %in% names(collections)) + matched <- as.numeric(collections[[matched_field]]) + matched +} + +#' @rdname collections_functions +#' +#' @export +collections_length <- function(collections) { + check_collections(collections) + return(length(collections$collections)) +} + +#' @rdname collections_functions +#' +#' @export +collections_fetch <- function(collections, ..., + progress = TRUE, + matched_field = NULL) { + check_collections(collections) + matched <- collections_matched(collections, matched_field) + # verify if progress bar can be shown + progress <- progress & + (!is.null(matched) && (collections_fetch(collections) < matched)) + if (progress) { + pb <- utils::txtProgressBar( + min = collections_length(collections), + max = matched, + style = 3 + ) + # close progress bar when exit + on.exit({ + if (progress) { + utils::setTxtProgressBar(pb, matched) + close(pb) + } + }) + } + # Initialize the items + next_collections <- collections + while (TRUE) { + # check if features is complete + if (!is.null(matched) && (collections_length(collections) == matched)) + break + # protect against infinite loop + if (!is.null(matched) && (collections_length(collections) > matched)) + .error(paste( + "Length of returned collections (%s) is different", + "from matched collections (%s)."), + collections_length(collections), matched) + next_collections <- tryCatch({ + collections_next(next_collections, ...) + }, next_error = function(e) NULL) + if (is.null(next_collections)) + break + collections$collections <- c(collections$collections, + next_collections$collections) + # update progress bar + if (progress) + utils::setTxtProgressBar(pb, length(next_collections)) + } + collections +} diff --git a/R/collections-query.R b/R/collections-query.R index 82d2a4f3..3f1515f1 100644 --- a/R/collections-query.R +++ b/R/collections-query.R @@ -14,11 +14,14 @@ #' Collection object #' } #' -#' @param q a `rstac_query` object expressing a STAC query +#' @param q a `rstac_query` object expressing a STAC query #' criteria. #' #' @param collection_id a `character` collection id to be retrieved. #' +#' @param limit an `integer` defining the maximum number of results +#' to return. If not informed, it defaults to the service implementation. +#' #' @seealso #' [get_request()], [post_request()], [items()] #' @@ -40,7 +43,7 @@ #' } #' #' @export -collections <- function(q, collection_id = NULL) { +collections <- function(q, collection_id = NULL, limit = NULL) { check_query(q, "stac") params <- list() subclass <- "collections" @@ -49,6 +52,8 @@ collections <- function(q, collection_id = NULL) { .error("Parameter `collection_id` must be a single value.") params$collection_id <- collection_id subclass <- "collection_id" + } else if (!is.null(limit)) { + params$limit <- limit } rstac_query( version = q$version, diff --git a/R/items-funs.R b/R/items-funs.R index 399392a9..b29ea091 100644 --- a/R/items-funs.R +++ b/R/items-funs.R @@ -30,8 +30,9 @@ #' \item `items_filter()`: selects only items that match some criteria #' (see details section). #' -#' \item `items_reap()`: extract key values by traversing all items -#' in a `doc_items` object. +#' \item `items_reap()`: traverses all items in a `doc_items` object and +#' extracts values based on the specified field path. It is useful for +#' retrieving nested elements from STAC items. #' #' \item `items_fields()`: lists field names inside an item. #' @@ -61,8 +62,9 @@ #' @param progress a `logical` indicating if a progress bar must be #' shown or not. Defaults to `TRUE`. #' -#' @param field a `character` with the names of the field to -#' get the subfields values. +#' @param field A `character` vector specifying the path to the +#' field from which to extract subfield values. +#' For example, `c("assets", "*")` will traverse all assets from each item. #' #' @param pick_fn a `function` used to pick elements from items #' addressed by `field` parameter. @@ -212,9 +214,13 @@ #' stac_search(collections = "CB4-16D-2", limit = 100, #' datetime = "2017-08-01/2018-03-01", #' bbox = c(-48.206, -14.195, -45.067, -12.272)) %>% -#' get_request() %>% items_fetch(progress = FALSE) +#' get_request() %>% +#' items_fetch(progress = FALSE) #' -#' stac_item %>% items_reap(field = c("properties", "datetime")) +#' stac_item %>% items_reap(c("properties", "datetime")) +#' +#' # Extract all asset URLs from each item +#' stac_item %>% items_reap(c("assets", "*"), \(x) x$href) #' #' stac_item %>% items_as_sf() #' diff --git a/R/preview-utils.R b/R/preview-utils.R index eac51640..10a6cab8 100644 --- a/R/preview-utils.R +++ b/R/preview-utils.R @@ -2,7 +2,7 @@ #' #' This is a helper function to plot preview assets #' (e.g. quicklook, thumbnail, rendered_preview). -#' Currently, only png and jpeg formats are supported. +#' Currently, only png, jpeg and jpg formats are supported. #' #' @param url image URL to be plotted. #' @@ -12,7 +12,7 @@ preview_plot <- function(url) { preview_check(url) img <- preview_read_file(url) - plot(1:10, ty = "n", axes = F, xlab = "", ylab = "") + plot(1:10, type = "n", axes = FALSE, xlab = "", ylab = "") grid::grid.raster(img) } @@ -39,6 +39,12 @@ preview_check <- function(url) { "This function requires `jpeg` package. Please, use", "install.packages('jpeg')." )) + , + jpg = if (!requireNamespace("jpeg", quietly = TRUE)) + .error(paste( + "This function requires `jpeg` package. Please, use", + "install.packages('jpeg')." + )) ) } @@ -55,7 +61,8 @@ preview_read_file <- function(url) { preview_switch( url, png = png::readPNG(temp_file), - jpeg = jpeg::readJPEG(temp_file) + jpeg = jpeg::readJPEG(temp_file), + jpg = jpeg::readJPEG(temp_file) ) } diff --git a/man/assets_functions.Rd b/man/assets_functions.Rd index 7f37e02a..936bc29c 100644 --- a/man/assets_functions.Rd +++ b/man/assets_functions.Rd @@ -33,6 +33,7 @@ assets_download( output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, download_fn = NULL ) @@ -42,6 +43,7 @@ assets_download( output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, create_json = FALSE, download_fn = NULL ) @@ -52,6 +54,7 @@ assets_download( output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, download_fn = NULL, create_json = TRUE, items_max = Inf, @@ -64,6 +67,7 @@ assets_download( output_dir = getwd(), overwrite = FALSE, ..., + use_gdal = FALSE, create_json = FALSE, download_fn = NULL ) @@ -123,6 +127,9 @@ if FALSE, a warning message is shown.} \item{...}{additional arguments. See details.} +\item{use_gdal}{a \code{logical} indicating if the file should be downloaded +by GDAL instead httr package.} + \item{download_fn}{a \code{function} to handle download of assets for each item to be downloaded. Using this function, you can change the hrefs for each asset, as well as the way download is done.} diff --git a/man/collections.Rd b/man/collections.Rd index f5a452ed..7dfc9f22 100644 --- a/man/collections.Rd +++ b/man/collections.Rd @@ -4,13 +4,16 @@ \alias{collections} \title{Endpoint functions} \usage{ -collections(q, collection_id = NULL) +collections(q, collection_id = NULL, limit = NULL) } \arguments{ \item{q}{a \code{rstac_query} object expressing a STAC query criteria.} \item{collection_id}{a \code{character} collection id to be retrieved.} + +\item{limit}{an \code{integer} defining the maximum number of results +to return. If not informed, it defaults to the service implementation.} } \value{ A \code{rstac_query} object with the subclass \code{collections} for diff --git a/man/collections_functions.Rd b/man/collections_functions.Rd new file mode 100644 index 00000000..b5df58b0 --- /dev/null +++ b/man/collections_functions.Rd @@ -0,0 +1,82 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/collections-funs.R +\name{collections_functions} +\alias{collections_functions} +\alias{collections_next} +\alias{collections_matched} +\alias{collections_length} +\alias{collections_fetch} +\title{Collections functions} +\usage{ +collections_next(collections, ...) + +collections_matched(collections, matched_field) + +collections_length(collections) + +collections_fetch(collections, ..., progress = TRUE, matched_field = NULL) +} +\arguments{ +\item{collections}{a \code{doc_collections} object.} + +\item{...}{additional arguments. See details.} + +\item{matched_field}{a \code{character} vector with the path +where is the number of collections returned.} + +\item{progress}{a \code{logical} indicating if a progress bar must be +shown or not. Defaults to \code{TRUE}.} +} +\value{ +\itemize{ +\item \code{collections_length()}: an \code{integer} value. + +\item \code{collections_matched()}: returns an \code{integer} value if the STAC web +server does support this extension. Otherwise returns \code{NULL}. + +\item \code{collections_fetch()}: a \code{doc_items} with all matched items. + +\item \code{collections_next()}: fetches a new page from STAC service. + +} +} +\description{ +These functions provide support to work with +\code{doc_collections}objects. + +\itemize{ +\item \code{collections_length()}: \ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +shows how many items there are in the \code{doc_items} object. + +\item \code{collections_matched()}: \ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +shows how many items matched the search criteria. + +\item \code{collections_fetch()}: \ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +request all STAC Items through pagination. + +\item \code{collections_next()}: \ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +fetches a new page from STAC service. + +} +} +\details{ +Ellipsis argument (\code{...}) appears in different items functions and +has distinct purposes: + +\itemize{ +\item \code{collections_fetch()} and \code{collections_next()}: ellipsis is used to +pass additional \code{httr} options to \link[httr:GET]{GET} method, such as +\link[httr:add_headers]{add_headers} or \link[httr:set_cookies]{set_cookies}. + +} +} +\examples{ +\dontrun{ +# doc_items object +stac("https://cmr.earthdata.nasa.gov/stac/LPCLOUD") |> + collections() |> + get_request() |> + collections_fetch() +} + +} diff --git a/man/items_functions.Rd b/man/items_functions.Rd index dd25fdce..1df1aa96 100644 --- a/man/items_functions.Rd +++ b/man/items_functions.Rd @@ -171,8 +171,9 @@ shown or not. Defaults to \code{TRUE}.} \item{filter_fn}{a \code{function} that receives an item that should evaluate a \code{logical} value.} -\item{field}{a \code{character} with the names of the field to -get the subfields values.} +\item{field}{A \code{character} vector specifying the path to the +field from which to extract subfield values. +For example, \code{c("assets", "*")} will traverse all assets from each item.} \item{pick_fn}{a \code{function} used to pick elements from items addressed by \code{field} parameter.} @@ -259,8 +260,9 @@ field of a \code{doc_items} or a \code{doc_item} object. \item \code{items_filter()}: selects only items that match some criteria (see details section). -\item \code{items_reap()}: extract key values by traversing all items -in a \code{doc_items} object. +\item \code{items_reap()}: traverses all items in a \code{doc_items} object and +extracts values based on the specified field path. It is useful for +retrieving nested elements from STAC items. \item \code{items_fields()}: lists field names inside an item. @@ -366,9 +368,13 @@ stac_item <- stac("https://brazildatacube.dpi.inpe.br/stac/") \%>\% stac_search(collections = "CB4-16D-2", limit = 100, datetime = "2017-08-01/2018-03-01", bbox = c(-48.206, -14.195, -45.067, -12.272)) \%>\% - get_request() \%>\% items_fetch(progress = FALSE) + get_request() \%>\% + items_fetch(progress = FALSE) -stac_item \%>\% items_reap(field = c("properties", "datetime")) +stac_item \%>\% items_reap(c("properties", "datetime")) + +# Extract all asset URLs from each item +stac_item \%>\% items_reap(c("assets", "*"), \(x) x$href) stac_item \%>\% items_as_sf() diff --git a/man/preview_plot.Rd b/man/preview_plot.Rd index 96ead0c5..4ce4a078 100644 --- a/man/preview_plot.Rd +++ b/man/preview_plot.Rd @@ -15,5 +15,5 @@ A rastergrob grob from package \code{grid}. \description{ This is a helper function to plot preview assets (e.g. quicklook, thumbnail, rendered_preview). -Currently, only png and jpeg formats are supported. +Currently, only png, jpeg and jpg formats are supported. }