diff --git a/DESCRIPTION b/DESCRIPTION index 23b7106b..5873833e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,6 +42,8 @@ Suggests: debugme, fansi, fs, + gh, + gitcreds, glue, htmlwidgets, mockery, @@ -54,6 +56,12 @@ Suggests: testthat (>= 3.2.0), tibble, webfakes (>= 1.1.5.9000), + withr (>= 2.1.1), +Remotes: + r-lib/pkgcache +Config/Needs/builder: + gh, + pkgsearch, withr (>= 2.1.1) Config/Needs/coverage: r-lib/asciicast, diff --git a/NAMESPACE b/NAMESPACE index 4a3e4a48..ea993c07 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,6 +32,8 @@ export(as_pkg_dependencies) export(current_config) export(current_r_platform) export(default_platforms) +export(ghr) +export(ghrepo) export(install_package_plan) export(is_valid_package_name) export(lib_status) @@ -41,6 +43,7 @@ export(new_pkg_installation_plan) export(new_pkg_installation_proposal) export(parse_pkg_ref) export(parse_pkg_refs) +export(pkg_build) export(pkg_dep_types) export(pkg_dep_types_hard) export(pkg_dep_types_soft) @@ -50,6 +53,7 @@ export(pkg_installation_plan) export(pkg_installation_proposal) export(pkg_name_check) export(pkg_rx) +export(repo) export(sysreqs_check_installed) export(sysreqs_db_list) export(sysreqs_db_match) diff --git a/R/builder.R b/R/builder.R new file mode 100644 index 00000000..e8279a23 --- /dev/null +++ b/R/builder.R @@ -0,0 +1,86 @@ + +#' Create a binary package from an installed package +#' +#' The built package will be in the current working directory. +#' +#' This function is currently experimental. +#' +#' @param pkg Package name. +#' @param library Library path. +#' @param flavor Platform flavor. Defaults to the `PKG_BUILD_FLAVOR` +#' environment variable. If not `NULL` or an empty string, then it is +#' appended to the platform string with a dash. +#' @param build_number An integer number that is added to the file name, +#' after the version number, to be able to have multiple builds for the +#' same package version. +#' @return Path to the built package. +#' +#' @export +#' @keywords internal + +pkg_build <- function(pkg, library = .libPaths()[1], + flavor = Sys.getenv("PKG_BUILD_FLAVOR"), + build_number = 1L) { + pkgdir <- file.path(library, pkg) + if (!dir.exists(pkgdir)) { + throw(pkg_error( + "Cannot find package {.pkg {pkg}} in library at {.path {library}}." + )) + } + platform <- pkgcache::current_r_platform() + if (nzchar(flavor %||% "")) { + platform <- paste0(platform, "-", flavor) + } + meta <- c( + RemoteBuildPlatform = platform, + GraphicsAPIVersion = pkgcache::get_graphics_api_version(), + InternalsId = pkgcache::get_internals_id() + ) + add_metadata(pkgdir, meta) + dsc <- desc::desc(file = pkgdir) + version <- dsc$get_field("Version") + rversion <- get_minor_r_version(getRversion()) + + sys <- sysname() + if (sys == "windows") { + install_md5_sums(pkg) + fn <- paste0( + pkg, "_", version, "_", + "b", build_number, "_", + "R", rversion, + if (nzchar(flavor %||% "")) paste0("_", flavor), + ".zip" + ) + zip::zip(fn, pkgdir, mode = "cherry-pick") + + } else { + ext <- if (sys == "mac") ".tgz" else ".tar.gz" + fn <- paste0( + pkg, "_", version, "_", + "b", build_number, "_", + "R", rversion, "_", + platform, + ext + ) + ffn <- file.path(normalizePath("."), fn) + old <- getwd() + on.exit(setwd(old), add = TRUE) + setwd(dirname(pkgdir)) + utils::tar(ffn, pkg, compression = "gzip", compression_level = 9) + } + + fn +} + +install_md5_sums <- function(pkgdir) { + old <- getwd() + on.exit(setwd(old), add = TRUE) + + setwd(pkgdir) + fns <- setdiff(dir(".", recursive = TRUE), "MD5") + md5 <- cli::hash_file_md5(fns) + writeLines( + paste0(md5, " *", fns), + "MD5" + ) +} diff --git a/R/gh-mirror.R b/R/gh-mirror.R new file mode 100644 index 00000000..f992a9e9 --- /dev/null +++ b/R/gh-mirror.R @@ -0,0 +1,403 @@ + +ghmirror <- local({ + + ghmirror_update <- function(pkg, + source_repo = "https://cran.r-project.org") { + cran_versions <- get_package_versions(pkg, source_repo) + atgh_versions <- get_github_versions(pkg) + + missing <- setdiff(cran_versions$version, atgh_versions$version) + + add_missing_versions( + pkg, + missing, + new_pkg = length(atgh_versions) == 0, + repo = source_repo + ) + + invisible(missing) + } + + # ----------------------------------------------------------------------- + # Internal functions + + cran_versions <- NULL + + get_all_cran_versions <- function(source_repo = "https://cran.r-project.org", + forget = FALSE) { + + if (!is.null(cran_versions) && !forget) { + return(cran_versions) + } + + r_archive <- pkgcache::cran_archive_cache$new( + cran_mirror = source_repo, + update_after = as.difftime(30, units = "mins") + ) + r_archive$check_update() + r_archive_pkgs <- r_archive$list() + + r_source <- pkgcache::cranlike_metadata_cache$new( + platforms = "source", + bioc = FALSE, + cran_mirror = source_repo, + repos = NULL, + update_after = as.difftime(30, units = "mins") + ) + r_source$check_update() + r_source_pkgs <- r_source$list() + + # we'll just drop versions with invalid version numbers, these are + # old packages only + badver <- is.na(package_version(r_archive_pkgs$version, strict=FALSE)) + r_archive_pkgs <- r_archive_pkgs[!badver,, drop = FALSE] + + res <- data_frame( + package = c(r_archive_pkgs$package, r_source_pkgs$package), + version = c(r_archive_pkgs$version, r_source_pkgs$version) + ) + + res <- res[order(tolower(res$package), package_version(res$version)), ] + cran_versions <<- res + + res + } + + get_package_versions <- function(pkg, + source_repo = "https://cran.r-project.org", + forget = FALSE) { + all <- get_all_cran_versions(source_repo, forget = forget) + res <- all[all$package == pkg,, drop = FALSE] + res + } + + get_github_versions <- function(pkg) { + github_versions <- tryCatch( + gh::gh( + "/repos/:owner/:repo/tags", + owner = "cran", + repo = pkg, + .limit = Inf, + .token = get_gh_token() + ), + error = function(e) list(), + warning = function(e) list() + ) + github_versions <- vapply(github_versions, "[[", FUN.VALUE = "", "name") + github_versions <- grep("R-", github_versions, value = TRUE, invert = TRUE) + + data_frame( + version = rev(github_versions) + ) + } + + get_gh_token <- function() { + token <- Sys.getenv("CRANATGH_GITHUB_TOKEN", NA_character_) + if (is.na(token)) token <- Sys.getenv("GITHUB_PAT", NA_character_) + if (is.na(token)) token <- Sys.getenv("GITHUB_TOKEN", NA_character_) + token + } + + add_missing_versions <- function(pkg, versions, new_pkg, repo) { + if (length(versions) == 0) return() + + oldwd <- getwd() + on.exit(setwd(oldwd), add = TRUE) + change_to_cranatgh_home() + + if (new_pkg) create_git_repo(pkg) + + if (!file.exists(pkg)) clone_git_repo(pkg) + + set_git_user(pkg) + + for (ver in versions) { + metadata <- add_missing_version(pkg, ver, repo) + } + + desc <- make_description(metadata) + + if (new_pkg) create_gh_repo(pkg, desc) + + push_to_github(pkg) + + if (!new_pkg) update_description(pkg, desc) + + invisible() + } + + make_description <- function(pkg) { + if (is.character(pkg)) { + pkg <- tryCatch( + pkgsearch::cran_package(pkg), + error = function(e) NULL + ) + + } else if (inherits(pkg, "description")) { + pkg <- list( + Package = pkg$get("Package"), + Title = pkg$get("Title"), + URL = pkg$get("URL"), + BugReports = pkg$get("BugReports") + ) + + } else { + stop("'pkg' must be a character scalar or a 'description' object") + } + + dsc <- paste( + sep = " ", + dont_break(":exclamation: This is a read-only mirror of the CRAN R package repository."), + dont_break( + pkg$Package, " \u2014 ", pkg$Title, + nullna_or(pkg$URL, paste0(". Homepage: ", pkg$URL)) + ), + nullna_or(pkg$BugReports, dont_break("Report bugs for this package: ", pkg$BugReports)) + ) + + # Limit is 350 characters, but be conservative + if (nchar(dsc) > 320) dsc <- paste0(substr(dsc, 1, 320), " ...") + + dsc + } + + add_missing_version <- function(package, version, repo) { + + proc <- cli::cli_process_start("Adding {.pkg {package}} {version}") + + ## Rename the .git directory. We'll need it later + file.rename(file.path(package, ".git"), "dot-git") + + ## Remove everything from the old version + unlink(package, recursive = TRUE, force = TRUE) + dir.create(package) + + ## Put the new version in place + tar_file <- get_package_tarball(package, version, repo) + untar(tar_file) + unlink(tar_file) + + ## Put back the .git directory + ## The unlink is for the occasional case when there is already + ## a .git directory in the package. This is junk anyway, and it + ## should not be there + unlink(file.path(package, ".git")) + file.rename("dot-git", file.path(package, ".git")) + + setwd(package) + on.exit(setwd(".."), add = TRUE) + + ## To prevent an error like "detected dubious ownership in repository" + if (.Platform$OS.type == "unix") { + system("chown -R `id -un`:`id -gn` .") + } + + ## Add all the new files + git("status") + git("add", "-A", ".") + git("status") + + ## Package information from DESCRIPTION + metadata <- desc::description$new() + maint <- metadata$get_maintainer() + auth <- metadata$get("Author") + + ## Commit the new version + date <- format_iso_8601(metadata$get("Date/Publication")) + git( + env = c("GIT_COMMITTER_DATE" = date), + "commit", + "--allow-empty", + "-m", paste0("version ", version), + "--date", date, + "--author", fix_maintainer(maint, auth) + ) + + git("tag", version) + + cli::cli_process_done(proc) + + metadata + } + + change_to_cranatgh_home <- function() { + home <- default_tree_location() + if (is.na(home)) dir.create(home <- tempfile()) + setwd(home) + } + + default_tree_location <- function() { + Sys.getenv("CRANATGH_TREES", NA_character_) + } + + create_git_repo <- function(path) { + dir.create(path) + wd <- getwd() + on.exit(setwd(wd), add = TRUE) + setwd(path) + cli::cli_alert_info("Creating git repo in {.path {path}}") + git("init", ".") + } + + set_git_user <- function(path, user = NULL, email = NULL) { + + user <- user %||% default_cranatgh_user() + email <- email %||% default_cranatgh_email() + + withr::with_dir(path, { + git("config", "--local", "user.name", user) + git("config", "--local", "user.email", email) + }) + } + + default_cranatgh_user <- function() { + Sys.getenv("CRANATGH_USER", "cran-robot") + } + + default_cranatgh_email <- function() { + Sys.getenv("CRANATGH_EMAIL", "csardi.gabor+cran@gmail.com") + } + + default_cranatgh_org <- function() { + Sys.getenv("CRANATGH_ORG", "cran") + } + + clone_git_repo <- function(pkg) { + url <- get_clone_url(pkg) + proc <- cli::cli_process_start("Cloning GitHub repo from {.url {safe_url(url)}}") + git("clone", url) + cli::cli_process_done(proc) + } + + safe_url <- function(url) { + sub("://[-:a-z0-9]+@", "://@", url) + } + + get_clone_url <- function(package) { + + owner <- default_cranatgh_org() + token <- get_gh_token() + token <- if (is.na(token)) "" else paste0(token, "@") + + sprintf("https://%sgithub.com/%s/%s.git", token, owner, package) + } + + git <- function(..., env = character(), timeout = 60 * 60) { + processx::run( + "git", + args = unlist(list(...)), + env = c(Sys.getenv(), env) + ) + } + + get_package_tarball <- function(package, version, repo) { + urls <- package_urls(package, version, repo) + ok <- FALSE + for (url in urls) { + dest_file <- basename(url) + tryCatch({ + curl::curl_download(url, dest_file) + ok <- TRUE + break + }, error = function(err) err) + } + + if (!ok) stop("Cannot download package ", package) + dest_file + } + + package_urls <- function(package, version, repo) { + # work around some mistakes, file names do not match the version number + if (package == "HTML" && version == "0.4") { + "https://cran.rstudio.com/src/contrib/Archive/HTML/HTML_0.4-1.tar.gz" + + } else if (package == "timeslab" && version == "1.0") { + "https://cran.r-project.org/src/contrib/Archive/timeslab/timeslab_1.0-1.tar.gz" + + } else { + c(sprintf("%s/src/contrib/%s_%s.tar.gz", repo, package, version), + sprintf("%s/src/contrib/Archive/%s/%s_%s.tar.gz", repo, + package, package, version)) + } + } + + fix_maintainer <- function(maint, auth) { + if (is.na(maint)) maint <- auth + maint <- sub("\\s*<", " <", maint) + + ## ': end of single quote + ## ": start of double quote + ## ': single quote (within double quotes) + ## ": end of double quote + ## ': start of single quote for the rest of the string + maint <- gsub("'", paste0("'", '"', "'", '"', "'"), maint) + + if (is.na(maint)) maint <- "??? " + if (!grepl("<.*>", maint)) maint <- paste0(maint, " ") + if (toupper(maint) == "ORPHANED") maint <- "ORPHANED " + maint + } + + dont_break <- function(...) { + gsub("\\s+", "\u00a0", paste0(...)) + } + + nullna_or <- function(x, expr) { + if (is.null(x) || (length(x) == 1 && is.na(x))) "" else expr + } + + push_to_github <- function(package, forced_push = FALSE) { + wd <- getwd() + on.exit(setwd(wd), add = TRUE) + setwd(package) + + add_gh_remote(package) + + proc <- cli::cli_process_start("Pushing {.pkg {package}} to GitHub") + current <- sub("* ", "", fixed = TRUE, system("git branch", intern = TRUE)) + git("push", "--tags", if (forced_push) "-f", "-u", "origin", current) + cli::cli_process_done(proc) + } + + add_gh_remote <- function(package) { + + current <- git("remote")$stdout + if (! grepl("\\borigin\\b", current)) { + git( + "remote", "add", "origin", + get_clone_url(package) + ) + } + } + + update_description <- function(package, + description = make_description(package)) { + + description <- clean_description(description) + + proc <- cli::cli_process_start("Updating repo description for {.pkg {package}}") + gh::gh("PATCH /repos/:owner/:repo", + owner = default_cranatgh_org(), + repo = package, + name = package, + description = description, + homepage = "", + .token = get_gh_token() + ) + cli::cli_process_done(proc) + } + + clean_description <- function(description) { + description <- unname(description) + description <- gsub("\n", " ", description) + description + } + + structure( + list( + .internal = environment(), + + update = ghmirror_update + ) + ) +}) diff --git a/R/gh-releases.R b/R/gh-releases.R new file mode 100644 index 00000000..e28ba476 --- /dev/null +++ b/R/gh-releases.R @@ -0,0 +1,461 @@ + +ghr <- local({ + + # ------------------------------------------------------------------------- + + ghr_list <- function(repo) { + synchronize(async_ghr_list(repo)) + } + + async_ghr_list <- function(repo) { + repo <- parse_slug(repo) + query <- glue::glue("{ + rateLimit { + cost + remaining + } + repository(owner: \"\", name: \"\") { + releases(last: 100) { + nodes { + id + name + createdAt + tagName + } + } + } + }", .open = "<", .close = ">") + + github_query(query)$ + then(function(resp) { + rls <- resp$obj$data$repository$releases$nodes + data_frame( + id = vcapply(rls, "[[", "id"), + name = vcapply(rls, "[[", "name"), + tag_name = vcapply(rls, "[[", "tagName"), + created_at = parse_iso_8601(vcapply(rls, "[[", "createdAt")) + ) + }) + } + + # ------------------------------------------------------------------------- + + ghr_get <- function(repo, tag) { + synchronize(async_ghr_get(repo, tag)) + } + + async_ghr_get <- function(repo, tag) { + prepo <- parse_slug(repo) + ep <- glue::glue("/repos/{prepo$owner}/{prepo$repo}/releases/tags/{tag}") + async_github_v3_query(ep)$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) + } + + # ------------------------------------------------------------------------- + + ghr_list_assets <- function(repo, tag) { + synchronize(async_ghr_list_assets(repo, tag)) + } + + async_ghr_list_assets <- function(repo, tag) { + + repo <- parse_slug(repo) + query <- glue::glue("{ + rateLimit { + cost + remaining + } + repository(owner: \"\", name: \"\") { + release(tagName: \"\") { + releaseAssets(last: 100) { + nodes { + id + name + downloadUrl + size + createdAt + updatedAt + contentType + } + } + } + } + }", .open = "<", .close = ">") + + github_query(query)$ + then(function(resp) { + asts <- resp$obj$data$repository$release$releaseAssets$nodes + data_frame( + id = vcapply(asts, "[[", "id"), + name = vcapply(asts, "[[", "name"), + download_url = vcapply(asts, "[[", "downloadUrl"), + size = viapply(asts, "[[", "size"), + created_at = parse_iso_8601(vcapply(asts, "[[", "createdAt")), + updated_at = parse_iso_8601(vcapply(asts, "[[", "updatedAt")), + content_type = vcapply(asts, "[[", "contentType") + ) + }) + } + + # ------------------------------------------------------------------------- + + ghr_add_asset <- function(repo, file, tag, name = basename(file)) { + invisible(synchronize(async_ghr_add_asset(repo, file, tag, name))) + } + + async_ghr_add_asset <- function(repo, file, tag, + name = basename(file)) { + + repo; file; tag; name + + async_ghr_delete_asset(repo, tag, name)$ + then(function(res) res$release$upload_url)$ + catch(async_http_404 = function(err) { + async_ghr_create(repo, tag)$ + then(function(rel) rel$upload_url) + })$ + then(function(upload_url) { + upload_url <- sub("[{].*[}]", "", upload_url) + prepo <- parse_slug(repo) + async_github_v3_query( + url = upload_url, + endpoint = "", + query = c(name = name), + method = "POST", + headers = c("Content-Type" = "application/octet-stream"), + file = file + ) + })$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) + } + + # ------------------------------------------------------------------------- + + ghr_delete_asset <- function(repo, tag, name) { + synchronize(async_ghr_delete_asset(repo, tag, name)) + } + + async_ghr_delete_asset <- function(repo, tag, name) { + prepo <- parse_slug(repo) + async_ghr_get(repo, tag)$ + then(function(res) { + release_id <- res$id + asset_names <- vcapply(res$assets, "[[", "name") + if (name %in% asset_names) { + asset_id <- res$assets[[match(name, asset_names)]]$id + ep <- glue::glue("/repos/{prepo$user}/{prepo$repo}/releases/assets/{asset_id}") + async_github_v3_query(ep, method = "DELETE")$ + then(function(resp) { + list( + release = res, + deleted = TRUE + ) + }) + } else { + list(release = res, deleted = FALSE) + } + }) + } + + # ------------------------------------------------------------------------- + + ghr_create <- function(repo, tag) { + invisible(synchronize(async_ghr_create(repo, tag))) + } + + async_ghr_create <- function(repo, tag, description = "", draft = FALSE, + prerelease = FALSE, + generate_release_notes = FALSE) { + prepo <- parse_slug(repo) + ep <- glue::glue("/repos/{prepo$owner}/{prepo$repo}/releases") + data <- tojson$write_str(list( + tag_name = tag, + name = paste0(prepo$repo, " ", tag), + body = description, + draft = draft, + prerelease = prerelease, + generate_release_notes = generate_release_notes + ), list(auto_unbox = TRUE)) + + async_github_v3_query( + ep, + data = data, + method = "POST" + )$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) + } + + # ------------------------------------------------------------------------- + + parse_slug <- function(slug) { + parts <- strsplit(slug, "/", fixed = TRUE)[[1]] + list(user = parts[1], owner = parts[1], repo = parts[2]) + } + + async_github_v3_query <- function(endpoint, query = NULL, + method = c("GET", "POST", "DELETE"), + headers = NULL, + data = NULL, + file = NULL, + url = NULL) { + method <- match.arg(method) + + headers <- update_named_vector(type_github_get_headers(), headers) + + base <- url %||% Sys.getenv("R_PKG_GITHUB_API_URL", "https://api.github.com") + query_str <- paste( + glue::glue("{curl::curl_escape(names(query))}={curl::curl_escape(query)}"), + collapse = "&" + ) + url <- paste0(base, endpoint, "?", query_str) + + px <- if (method == "GET") { + http_get(url, headers = headers) + + } else if (method == "POST") { + if (is.null(data) + is.null(file) != 1) { + throw(pkg_error( + "Must specify exactly of {.arg data} and {.arg file} for POST requests." + )) + } + if (is.null(data)) data <- readBin(file, "raw", file.size(file)) + http_post(url, data, headers = headers) + } else if (method == "DELETE") { + http_delete(url, headers = headers) + } + + px$ + then(http_stop_for_status) + } + + # github_query() is in type-github.R + + # ----------------------------------------------------------------------- + # Exported functions + + structure( + list( + .internal = environment(), + + async_add_asset = async_ghr_add_asset, + async_delete_asset = async_ghr_delete_asset, + async_create = async_ghr_create, + async_get = async_ghr_get, + async_list = async_ghr_list, + async_list_assets = async_ghr_list_assets, + + add_asset = ghr_add_asset, + delete_asset = ghr_delete_asset, + create = ghr_create, + get = ghr_get, + list = ghr_list, + list_assets = ghr_list_assets + ) + ) +}) + +# ------------------------------------------------------------------------- + +#' GitHub Releases +#' +#' Functions to query and manipulate GitHub releases. These functions are +#' currently experimental. +#' +#' @details +#' +#' ## List releases +#' +#' ### Description +#' +#' `ghr$list()` lists the last 100 releases for a GitHub repository. +#' `ghr$async_list()` is the async version of `ghr$list()`. +#' +#' ### Usage +#' ``` +#' ghr$list(repo) +#' ghr$async_list(repo) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `"cran/cli"`. +#' +#' ### Value +#' +#' Data frame with columns: +#' * `id`: release id, +#' * `name`: release name, usually the version number, possibly with +#' a `v` prefix: `3.6.1` or `v3.6.1`, but can be different. +#' * `tag_name`: usually the same as `name`. +#' * `created_at`: `POSIXct` vector. +#' +# ------------------------------------------------------------------------- +#' +#' ## Get information about a release +#' +#' ### Description +#' +#' `ghr$get()` downloads information about a release, including +#' release assets. +#' +#' `ghr$async_get` is the async version of `ghr$get`. +#' +#' ### Usage +#' ``` +#' ghr$get(repo, tag) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `"cran/cli"`. +#' * `tag`: tag to get. +#' +#' ### Value +#' +#' Named list, see +#' +#' for the entries. +#' +# ------------------------------------------------------------------------- +#' +#' ## List assets of a release +#' +#' ### Description +#' +#' `ghr$list_assets()` lists the last 100 assets of a release. +#' +#' `ghr$async_list_assets()` is the async version of `ghr$list_assets()` +#' +#' ### Usage +#' ``` +#' ghr$list_assets(repo, tag) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `"cran/cli"`. +#' * `tag`: tag to query. +#' +#' ### Value +#' +#' Data frame with columns: +#' * `id`: asset id, +#' * `name`: file name of the asset, +#' * `download_url`: download URL, +#' * `size`: size in bytes, +#' * `created_at`: `POSIXct` vector, +#' * `updated_at`: `POSXct` vector, +#' * `content_type`: content type of asset. +#' +# ------------------------------------------------------------------------- +#' +#' ## Add a release asset +#' +#' ### Description +#' +#' `ghr$add_asset()` adds an asset to a GitHub release. +#' +#' `ghr$async_add_asset()` is the async version of `ghr$add_asset()`. +#' +#' ### Usage +#' ``` +#' ghr%add_asset(repo, file, tag, name = basename(file)) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `cran/cli`. +#' * `file`: path to file to upload as an asset. +#' * `tag`: tag name to add the asset to. It must exist on GitHub. +#' * `name`: file name of the asset in the release. +#' +#' ### Details +#' +#' If an asset with the same name already exists, then that will be deleted +#' first. +#' +#' ### Value +#' +#' Response from GitHub as a named list. See +#' +#' for the structure. +#' +# ------------------------------------------------------------------------- +#' ## Delete a release asset +#' +#' ### Description +#' +#' `ghr$delete_asset()` deleted a release asset. +#' +#' `ghr$async_delete_asset()` is an async version of `ghr$delete_asset()`. +#' +#' ### Usage +#' ``` +#' ghr$delete_asset(repo, tag, name) +#' ghr$async_delete_asset(repo, tag, name) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `cran/cli`. +#' * `tag`: tag name to create a release for. It must exist on GitHub. +#' * `name`: name of the asset. +#' +#' ### Value +#' +#' A list with entries: +#' * `release`: a list with the data about the release, before the +#' deletion. It has the same format as the return value of `ghr$get()`. +#' * `deleted`: `TRUE` if the asset was deleted. `FALSE` if the asset +#' did not exist. +#' +# ------------------------------------------------------------------------- +#' +#' ## Create a GitHub release +#' +#' ### Description +#' +#' `ghr$create()` creates a GitHub release from a tag. +#' +#' `ghr$async_create()` is an async version of `ghr$create()`. +#' +#' ### Usage +#' ``` +#' ghr$create( +#' repo, +#' tag, +#' description = "", +#' draft = FALSE, +#' prerelease = FALSE, +#' generage_release_notes = FALSE +#' ) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: repository slug, e.g. `cran/cli`. +#' * `tag`: tag name to create a release for. It must exist on GitHub. +#' * `description`: release description. +#' * `draft`: whether to create a draft release. +#' * `prerelease`: whether to create a prerelease. +#' * `generate_release_notes`: whether to auto-generate release notes. +#' +#' ### Value +#' +#' Response from GitHub as a named list. See +#' +#' for the structure. +#' +# ------------------------------------------------------------------------- +#' +#' @name ghr +#' @keywords internal +#' @export + +ghr diff --git a/R/gh-repo.R b/R/gh-repo.R new file mode 100644 index 00000000..e2bfc373 --- /dev/null +++ b/R/gh-repo.R @@ -0,0 +1,339 @@ +ghrepo <- local({ + + ghrepo_update <- function(repo, subdir, release_org = "cran", + source_repo = "https://cran.r-project.org", + ignore_build_errors = TRUE, packages = NULL) { + + subdir <- sub("/+$", "", subdir) + if (endsWith(subdir, "src/contrib")) { + subdir <- sub("src/contrib$", "", subdir) + } + + cli::cli_h2("Collect information") + source_pkgs <- get_cran_packages(source_repo) + mirror_pkgs <- get_mirror_packages(repo, subdir) + + to_build <- get_outdated_packages(source_pkgs, mirror_pkgs, packages) + if (length(to_build) == 0) return(invisible(character())) + + cli::cli_h2("Install packages") + dir.create(lib <- tempfile("pkgdepends-lib-")) + inst <- install_pkgs( + c(to_build, if (ignore_build_errors) "*=?ignore-build-errors"), + library = lib, + source_repo = source_repo, + repo = repo, + subdir = subdir + ) + inst <- keep_updated(inst, mirror_pkgs) + # I don't think this can happen.... + if (nrow(inst) == 0) return(invisible(character())) + + cli::cli_h2("Build binary packages") + inst <- build_pkgs(inst, lib) + # drop the ones that failed to build + inst <- inst[!is.na(inst$built_path), ] + + cli::cli_par() + cli::cli_h2("Update package mirror at GH (if needed)") + # make sure these packages are mirrored on GH + for (i in seq_along(inst$package)) { + ghmirror$update(inst$package[[i]], source_repo = source_repo) + } + + if (nrow(inst) > 0) { + cli::cli_par() + cli::cli_h2("Upload new release assets to GH") + upload_releases(inst, release_org) + } + + if (nrow(inst) > 0) { + cli::cli_par() + cli::cli_h2("Update repository metadata at GH") + # unfortunate name collision + repo_slug <- repo + rm(repo) + contrib_url <- utils::contrib.url(repos = "", type = .Platform$pkgType) + repo$update_gh( + repo_slug, + paste0(subdir, contrib_url), + inst$built_path, + release_org = release_org + ) + } + + cli::cli_par() + cli::cli_h2("Build report") + print_report(inst, mirror_pkgs) + + invisible(inst) + } + + # ----------------------------------------------------------------------- + # Internal functions + + parse_build_number <- function(x) { + x <- basename(x) + mch <- re_match(x, "_b(?[0-9]+)_") + build <- as.integer(mch$build) + 1L + build[is.na(build)] <- 1L + build + } + + get_cran_packages <- function(repo) { + proc <- cli::cli_process_start("Getting packages from {.url {repo}}.") + r_source <- suppressMessages(pkgcache::cranlike_metadata_cache$new( + platforms = "source", + bioc = FALSE, + cran_mirror = repo, + repos = NULL, + update_after = as.difftime(30, units = "mins") + )) + suppressMessages(r_source$update()) + res <- r_source$list() + + cli::cli_process_done(proc) + res + } + + get_mirror_repo_url <- function(repo, subdir) { + subdir <- sub("/src/contrib/?$", "", subdir) + paste0( + "https://raw.githubusercontent.com/", + repo, + "/main/", + subdir + ) + } + + get_mirror_packages <- function(repo, subdir) { + proc <- cli::cli_process_start( + "Getting packages from {.emph {repo}/{subdir}}." + ) + platform <- if (.Platform$pkgType == "source") "source" else pkgcache::current_r_platform() + r_mirror <- suppressMessages(pkgcache::cranlike_metadata_cache$new( + platforms = platform, + bioc = FALSE, + cran_mirror = get_mirror_repo_url(repo, subdir), + repos = NULL, + update_after = as.difftime(1, units = "mins") + )) + suppressMessages(r_mirror$update()) + res <- r_mirror$list() + + cli::cli_process_done(proc) + res + } + + get_outdated_packages <- function(source_pkgs, mirror_pkgs, extra) { + # these are included already + extra <- setdiff(extra, mirror_pkgs$package) + + # TODO: update for breaking changes in R as well + to_build <- intersect(mirror_pkgs$package, source_pkgs$package) + r_mirror_ver <- mirror_pkgs$version[match(to_build, mirror_pkgs$package)] + r_source_ver <- source_pkgs$version[match(to_build, source_pkgs$package)] + to_build <- to_build[package_version(r_mirror_ver) < r_source_ver] + + res <- c(to_build, extra) + if (length(res) == 0) { + cli::cli_alert_info("All packages are current, no updates are needed") + } else { + cli::cli_alert_info( + "Need to build {length(res)} package{?s}: {.pkg {res}}." + ) + } + res + } + + install_pkgs <- function(pkgs, library, source_repo, repo, subdir) { + opt <- options(repos = c( + RHUB = get_mirror_repo_url(repo, subdir), + CRAN = source_repo) + ) + on.exit(options(opt), add = TRUE) + prop <- new_pkg_installation_proposal( + pkgs, + config = list( + library = library, + platforms = c(current_r_platform(), "source"), + use_bioconductor = FALSE, + metadata_update_after = as.difftime(5, units = "mins") + ) + ) + prop$set_solve_policy("upgrade") + prop$solve() + prop$show_solution() + prop$download() + prop$install_sysreqs() + inst <- prop$install() + + # Drop packages that were kept in a different library. These are + # recommended packages currently + was_inst <- inst$package[inst$type == "installed"] + drop <- setdiff(was_inst, dir(library)) + if (length(drop) > 0) { + inst <- inst[! inst$package %in% drop,, drop = FALSE ] + } + inst + } + + keep_updated <- function(inst, mirror_pkgs) { + # we installed these packages, now we update packages that are not in + # the mirror, or we just built a newer version for them. + # TODO: update for breaking changes in R as well + to_add <- setdiff(inst$package, mirror_pkgs$package) + mirrored <- intersect(inst$package, mirror_pkgs$package) + inst_ver <- inst$version[match(mirrored, inst$package)] + mirror_ver <- mirror_pkgs$version[match(mirrored, mirror_pkgs$package)] + to_update <- mirrored[package_version(inst_ver) > mirror_ver] + todo <- c(to_add, to_update) + inst <- inst[match(todo, inst$package),, drop = FALSE] + + # get build numbers + inst$buildnum <- rep(1L, nrow(inst)) + inst$buildnum[todo %in% mirrored] <- + parse_build_number(mirror_pkgs$target[match(todo[todo %in% mirrored], mirror_pkgs$package)]) + inst + } + + build_pkgs <- function(inst, library) { + files <- rep(NA_character_, nrow(inst)) + for (i in seq_along(inst$package)) { + if (length(inst$build_error[[i]]) && + !identical(inst$build_error[[i]], FALSE)) { + cli::cli_alert_warning( + "Failed to build package {.pkg {inst$package[[i]]}}." + ) + } else { + proc <- cli::cli_process_start( + "Building binary for package {.pkg {inst$package[[i]]}}." + ) + files[i] <- pkg_build( + inst$package[[i]], + library = library, + build_number = inst$buildnum[[i]] + ) + cli::cli_process_done(proc) + } + } + inst$built_path <- files + inst + } + + upload_releases <- function(inst, release_org) { + # upload packages to releases + for (i in seq_along(inst$package)) { + slug <- paste0(release_org, "/", inst$package[[i]]) + ver <- inst$version[[i]] + rels <- try(ghr$list(slug)) + if (inherits(rels, "try-error")) { + Sys.sleep(5) + rels <- ghr$list(slug) + } + if (!ver %in% rels$tag_name) { + cli::cli_alert_info("Creating GH release {slug} {ver}.") + ghr$create(slug, ver) + } + cli::cli_alert_info("Adding release asset for {slug} {ver}.") + tryCatch( + ghr$add_asset(slug, inst$built_path[i], ver), + error = function(err) { + cli::cli_alert_info("Try adding release asset again.") + Sys.sleep(60) + ghr$add_asset(slug, inst$built_path[i], ver) + } + ) + Sys.sleep(5) + } + } + + print_report <- function(inst, mirror_pkgs) { + pkg <- format(inst$package) + oldver <- ifelse( + inst$package %in% mirror_pkgs$package, + mirror_pkgs$version[match(inst$package, mirror_pkgs$package)], + "" + ) + + inst$build_time <- inst$build_time %||% NA_real_ + inst$build_time <- as.double(inst$build_time) + inst$build_time <- as.difftime(inst$build_time, units = "secs") + cols <- data_frame( + package = inst$package, + "old" = oldver, + "-" = "->", + "new" = inst$version, + "build" = paste0("b", inst$buildnum), + "build time" = vcapply(inst$build_time, format_time$pretty_dt) + ) + + print_table(cols) + } + + print_table <- function(cols) { + cols <- as.list(cols) + for (i in seq_along(cols)) { + if (names(cols)[i] == "-") names(cols)[i] <- "" + cols[[i]] <- format(c(names(cols)[i], "", cols[[i]])) + if (names(cols)[i] != "") { + cols[[i]][2] <- strrep("-", nchar(cols[[i]][1])) + } + } + writeLines(do.call(paste, cols)) + } + + # ----------------------------------------------------------------------- + + structure( + list( + .internal = environment(), + + update = ghrepo_update + ) + ) +}) + +# ------------------------------------------------------------------------- + +#' Update a CRAN-like repository of binary packages at GitHub +#' +#' These functions are currently experimental. +#' +#' @details +#' +#' ## Update a CRAN-like repository of binary packages at GitHub +#' +#' `ghrepo$update()` updates a binary package mirror. +#' +#' ### Usage +#' ``` +#' ghrepo$update( +#' repo, +#' subdir, +#' release_org = "cran", +#' source_repo = "https://cran.r-project.org", +#' packages = NULL +#' ) +#' ``` +#' +#' ### Arguments +#' +#' * `repo`: GitHub slug, e.g. `r-hub/repos`. +#' * `subdir`: subdirectory in the GitHub repository, where the R package +#' metadata should be updated. It must exist in the repository. +#' If it does not have `PACKAGES*` files, then they will be created. +#' * `release_org`: GitHub organization or user name where the packages +#' will be published as releases. +#' * `source_repo`: A CRAN-like repository, where source packages are +#' taken from. +#' * `packages`: A character vector of package names to add to the binary +#' repository, in addition to updating the ones that are already there. +#' +# ------------------------------------------------------------------------- +#' +#' @name ghrepo +#' @keywords internal +#' @export + +ghrepo diff --git a/R/ghcr.R b/R/ghcr.R new file mode 100644 index 00000000..98c6cfce --- /dev/null +++ b/R/ghcr.R @@ -0,0 +1,320 @@ + +ghcr_const_annotations <- list( + com.github.package.type = "r_package" +# org.opencontainers.image.authors = "Gabor Csardi", +# org.opencontainers.image.url = "https://github.com/r-lib/pak", +# org.opencontainers.image.documentation = "https://pak.r-lib.org/", +# org.opencontainers.image.source = "https://github.com/r-lib/pak", +# org.opencontainers.image.title = "pak R package", +# org.opencontainers.image.description = "Package manager for R", +# org.opencontainers.image.licenses = "GPL-3" +) + +ghcr_const_annotations_js <- function() { + paste0( + glue::glue('"{names(ghcr_const_annotations)}": "{ghcr_const_annotations}"'), + collapse = ",\n " + ) +} + +ghcr_get_package_data <- function(path) { + if (!file.exists(path)) { + throw(pkg_error("{.path {path}} does not exist.")) + } + + plt <- pkgcache::current_r_platform_data() + if (plt$os != "linux" && !grepl("^linux-", plt$os)) { + throw(pkg_error( + "{.fn ghcr_push_package} only works on Linux.", + "i" = "It does not work on this platform ({.var {plt$platform}}) yet." + )) + } + + chain_error( + dsc <- desc::desc(path), + pkg_error( + "Cannot read {.code DESCRIPTION} from {.file {path}}.", + "i" = "A valid R package must have a {.code DESCRIPTION} file." + ) + ) + + chain_error( + built <- dsc$get_built(), + pkg_error( + "Cannot find {.code Built} field in package at {.file {path}}", + "i" = "A binary R package must have a {.code Built} field in + {.code DESCRIPTION}." + ) + ) + + sha256 <- cli::hash_file_sha256(path) + data_frame( + path = path, + md5 = cli::hash_file_md5(path), + sha256 = sha256, + digest = paste0("sha256:", sha256), + image_title = paste("R package", dsc$get_field("Package")), + size = file.size(path), + buildtime = format_iso_8601(built$Date), + r_version = as.character(built$R), + r_platform = built$Platform, + package_name = dsc$get_field("Package"), + package_version = dsc$get_field("Version"), + license = dsc$get_field("License"), + arch = ghcr_canonize_arch(built$Platform), + os = ghcr_canonize_os(built$Platform) + ) +} + +ghcr_create_oci_repo <- function(pkgs, repo = NULL, oci_path = NULL, + cleanup = TRUE) { + + if (nrow(pkgs) > 1) { + throw(pkg_error("Multiple package versions are not supported yet.")) + } + + repo <- repo %||% Sys.getenv("GITHUB_REPOSITORY", NA_character_) + if (is.na(repo)) { + throw(pkg_error("Cannot detect GitHub repository")) + } + + oci_path <- oci_path %||% tempfile() + mkdirp(oci_path) + blob_path <- file.path(oci_path, "blobs", "sha256") + mkdirp(blob_path) + + # Create oci-layout file + cat( + '{"imageLayoutVersion": "1.0.0"}\n', + file = file.path(oci_path, "oci-layout") + ) + + # Copy package files to blobs + file.copy(pkgs$path, file.path(blob_path, pkgs$sha256)) + + # Config is dummy now + pkgs$image_config <- "{}" + pkgs$image_config_hash <- cli::hash_sha256(pkgs$image_config) + write_files(pkgs$image_config, file.path(blob_path, pkgs$image_config_hash)) + + # Create manifests + pkgs$manifest <- ghcr_image_manifest(pkgs) + pkgs$manifest_hash <- cli::hash_sha256(pkgs$manifest) + write_files(pkgs$manifest, file.path(blob_path, pkgs$manifest_hash)) + + # Create index + imidx <- ghcr_image_index(pkgs) + imidx_hash <- cli::hash_sha256(imidx) + write_files(imidx, file.path(blob_path, imidx_hash)) + + # index.json + idxjs <- jsonlite::prettify(glue::glue( + '{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:<>", + "size": <> + } + ] + }', .open = "<<", .close = ">>" + )) + write_files(idxjs, file.path(oci_path, "index.json")) + + # policy.json + policy_file <- file.path(oci_path, "policy.json") + policy <- jsonlite::prettify( + '{ + "default": [ + { + "type": "insecureAcceptAnything" + } + ], + "transports": + { + "docker-daemon": + { + "": [{"type":"insecureAcceptAnything"}] + } + } + }') + write_files(policy, policy_file) + + oci_path +} + +ghcr_uri <- function() { + Sys.getenv( + "PAK_GHCR_URI", + "docker://ghcr.io/r-lib/pak" + ) +} + +ghcr_user <- function() { + Sys.getenv("PAK_GHCR_USER", "gaborcsardi") +} + +ghcr_token <- function() { + Sys.getenv("PAK_GHCR_TOKEN", gitcreds::gitcreds_get()$password) +} + +ghcr_push_oci_repo <- function(oci_path, ghcr_tag = NULL) { + if (!file.exists(oci_path)) { + throw(pkg_error("Could not find OCI repository at {.path {oci_path}}.")) + } + ghcr_tag <- ghcr_tag %||% "TODO" + + skopeo <- check_skopeo() + oci_path <- normalizePath(oci_path, winslash = "/") + policy_file <- file.path(oci_path, "policy.json") + args <- c( + "copy", + "--all", + "--retry-times", 20, + "--preserve-digests", + "--policy", policy_file, + paste0("--dest-creds=", ghcr_user(), ":", ghcr_token()), + paste0("oci:", oci_path), + paste0(ghcr_uri(), ":", ghcr_tag) + ) + "TODO" +} + +ghcr_canonize_arch <- function(platform) { + arch <- strsplit(platform, "-", fixed = TRUE)[[1]][1] + c("aarch64" = "arm64", "x86_64" = "amd64")[[arch]] +} + +ghcr_canonize_os <- function(platform) { + os <- strsplit(platform, "-", fixed = TRUE)[[1]][3] + if (substr(os, 1, 6) == "darwin") os <- "darwin" + if (substr(os, 1, 5) == "mingw") os <- "windows" + if (substr(os, 1, 7) == "solaris") os <- "solaris" + os +} + +ghcr_image_index <- function(pkgs) { + tmpl <- ' + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:<>", + "size": <>, + "platform": { + "architecture": "<>", + "os": "<>", + "os.version": "R <>", + "r_version": "<>", + "r_platform": "<>" + }, + "annotations": { + "org.opencontainers.image.ref.name": "<>--<>--<>", + "io.r-hub.package.digest": "<>" + } + }' + mnfts <- glue::glue_data(pkgs, tmpl, .open = "<<", .close = ">>") + + buildtime <- max(pkgs$buildtime) + package_name <- unique(package_name) + if (length(package_name) != 1) { + throw(pkg_error( + "Cannot push different packages at the same time.", + "i" = "Found packages {.pkg {package_name}}." + )) + } + + package_version <- unique(pkgs$package_version) + if (length(package_version) != 1) { + throw(pkg_error( + "Non-matching package versions when pushing package {.pkg {package_name}}.", + "i" = "All package versions must be the same.", + "i" = "Found package versions: {.var {package_version}}." + )) + } + + tmpl2 <- ' + { + "schemaVersion": 2, + "manifests": [ + <> + ], + "annotations": { + <>, + "org.opencontainers.image.created": "<>", + "org.opencontainers.image.version": "<>" + } + }' + + jsonlite::prettify(glue::glue(tmpl2, .open = "<<", .close = ">>")) +} + +ghcr_image_manifest <- function(pkgs) { + tmpl <- ' + { + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:<>", + "size": <> + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "<>", + "size": <>, + "annotations": { + "org.opencontainers.image.title": "<>" + } + } + ], + "annotations": { + <>, + "org.opencontainers.image.created": "<>", + "org.opencontainers.image.version": "<>" + } + }' + + glue::glue_data(pkgs, tmpl, .open = "<<", .close = ">>") +} + + +write_file <- function(txt, path) { + out <- file(path, open = "wb") + on.exit(close(out), add = TRUE) + cat(txt, file = out, sep = "") +} + +write_files <- function(txts, paths) { + invisible(mapply(write_file, txts, paths)) +} + +find_skopeo <- function() { + path <- Sys.which("skopeo") + if (path != "") return(path) + if (file.exists(cand <- "/usr/local/bin/skopeo")) return(cand) + if (file.exists(cand <- "/opt/homebrew/bin/skopeo")) return(cand) + throw(pkg_error("Need skopeo to push packages.")) +} + +skopeo_version <- function() { + skopeo <- find_skopeo() + out <- processx::run(skopeo, "--version") + re_ver <- "[ ]([0-9]+[.][0-9]+[.][0-9]+)" + if (!grepl(re_ver, out$stdout)) stop("Cannot determine skopeo version") + mch <- regexpr(re_ver, out$stdout, perl = TRUE) + beg <- attr(mch, "capture.start")[1] + end <- beg + attr(mch, "capture.length")[1] - 1L + package_version(substr(out$stdout, beg, end)) +} + +check_skopeo <- function() { + skopeo <- find_skopeo() + skopeo_ver <- skopeo_version() + if (skopeo_ver < "1.6.0") { + throw(pkg_error( + "Need at least skopeo 1.6.0 to push packages", + "i" = "Found skopeo {.emph {skopoe_ver}} at {.path {skopeo}}." + )) + } + skopeo[[1]] +} diff --git a/R/install-plan.R b/R/install-plan.R index 76667b31..71c29d73 100644 --- a/R/install-plan.R +++ b/R/install-plan.R @@ -705,28 +705,32 @@ stop_task_build <- function(state, worker) { state$plan$build_done[[pkgidx]] <- TRUE state$plan$build_time[[pkgidx]] <- time - state$plan$build_error[[pkgidx]] <- ! success state$plan$build_stdout[[pkgidx]] <- worker$stdout state$plan$worker_id[[pkgidx]] <- NA_character_ if (success) { - # do nothing - } else if (ignore_error) { - # upstream will probably fail as well, but march on, neverthelesss - state$plan$install_done[[pkgidx]] <- TRUE - ## Need to remove from the dependency list - state$plan$deps_left <- lapply(state$plan$deps_left, setdiff, pkg) + # a bit silly, but for compatibility... + state$plan$build_error[[pkgidx]] <- FALSE } else { - throw(pkg_error( - "Failed to build source package {.pkg {pkg}}.", - .data = list( - package = pkg, - version = version, - stdout = worker$stdout, - time = time - ), - .class = "package_build_error" - )) + build_error <- list( + package = pkg, + version = version, + stdout = worker$stdout, + time = time + ) + state$plan$build_error[[pkgidx]] <- build_error + if (ignore_error) { + # upstream will probably fail as well, but march on, neverthelesss + state$plan$install_done[[pkgidx]] <- TRUE + ## Need to remove from the dependency list + state$plan$deps_left <- lapply(state$plan$deps_left, setdiff, pkg) + } else { + throw(pkg_error( + "Failed to build source package {.pkg {pkg}}.", + .data = build_error, + .class = "package_build_error" + )) + } } if (success && !is.null(state$cache) && !is_true_param(prms, "nocache")) { diff --git a/R/iso-date.R b/R/iso-date.R new file mode 100644 index 00000000..687af22a --- /dev/null +++ b/R/iso-date.R @@ -0,0 +1,144 @@ + +milliseconds <- function(x) as.difftime(as.numeric(x) / 1000, units = "secs") +seconds <- function(x) as.difftime(as.numeric(x), units = "secs") +minutes <- function(x) as.difftime(as.numeric(x), units = "mins") +hours <- function(x) as.difftime(as.numeric(x), units = "hours") +days <- function(x) as.difftime(as.numeric(x), units = "days") +weeks <- function(x) as.difftime(as.numeric(x), units = "weeks") +wday <- function(x) as.POSIXlt(x, tz = "UTC")$wday + 1 +with_tz <- function(x, tzone = "") as.POSIXct(as.POSIXlt(x, tz = tzone)) +ymd <- function(x) as.POSIXct(x, format = "%Y %m %d", tz = "UTC") +yj <- function(x) as.POSIXct(x, format = "%Y %j", tz = "UTC") + +parse_iso_8601 <- function(dates, default_tz = "UTC") { + if (default_tz == "") default_tz <- Sys.timezone() + dates <- as.character(dates) + match <- re_match(dates, iso_regex) + matching <- !is.na(match$.match) + result <- rep(.POSIXct(NA_real_, tz = ""), length.out = length(dates)) + result[matching] <- parse_iso_parts(match[matching, ], default_tz) + class(result) <- c("POSIXct", "POSIXt") + with_tz(result, "UTC") +} + +parse_iso_parts <- function(mm, default_tz) { + + num <- nrow(mm) + + ## ----------------------------------------------------------------- + ## Date first + + date <- .POSIXct(rep(NA_real_, num), tz = "") + + ## Years-days + fyd <- is.na(date) & mm$yearday != "" + date[fyd] <- yj(paste(mm$year[fyd], mm$yearday[fyd])) + + ## Years-weeks-days + fywd <- is.na(date) & mm$week != "" & mm$weekday != "" + date[fywd] <- iso_week(mm$year[fywd], mm$week[fywd], mm$weekday[fywd]) + + ## Years-weeks + fyw <- is.na(date) & mm$week != "" + date[fyw] <- iso_week(mm$year[fyw], mm$week[fyw], "1") + + ## Years-months-days + fymd <- is.na(date) & mm$month != "" & mm$day != "" + date[fymd] <- ymd(paste(mm$year[fymd], mm$month[fymd], mm$day[fymd])) + + ## Years-months + fym <- is.na(date) & mm$month != "" + date[fym] <- ymd(paste(mm$year[fym], mm$month[fym], "01")) + + ## Years + fy <- is.na(date) + date[fy] <- ymd(paste(mm$year, "01", "01")) + + ## ----------------------------------------------------------------- + ## Now the time + + th <- mm$hour != "" + date[th] <- date[th] + hours(mm$hour[th]) + + tm <- mm$min != "" + date[tm] <- date[tm] + minutes(mm$min[tm]) + + ts <- mm$sec != "" + date[ts] <- date[ts] + seconds(mm$sec[ts]) + + ## ----------------------------------------------------------------- + ## Fractional time + + frac <- as.numeric(sub(",", ".", mm$frac)) + + tfs <- !is.na(frac) & mm$sec != "" + date[tfs] <- date[tfs] + milliseconds(round(frac[tfs] * 1000)) + + tfm <- !is.na(frac) & mm$sec == "" & mm$min != "" + sec <- trunc(frac[tfm] * 60) + mil <- round((frac[tfm] * 60 - sec) * 1000) + date[tfm] <- date[tfm] + seconds(sec) + milliseconds(mil) + + tfh <- !is.na(frac) & mm$sec == "" & mm$min == "" + min <- trunc(frac[tfh] * 60) + sec <- trunc((frac[tfh] * 60 - min) * 60) + mil <- round((((frac[tfh] * 60) - min) * 60 - sec) * 1000) + date[tfh] <- date[tfh] + minutes(min) + seconds(sec) + milliseconds(mil) + + ## ----------------------------------------------------------------- + ## Time zone + + ftzpm <- mm$tzpm != "" + m <- ifelse(mm$tzpm[ftzpm] == "+", -1, 1) + ftzpmh <- ftzpm & mm$tzhour != "" + date[ftzpmh] <- date[ftzpmh] + m * hours(mm$tzhour[ftzpmh]) + ftzpmm <- ftzpm & mm$tzmin != "" + date[ftzpmm] <- date[ftzpmm] + m * minutes(mm$tzmin[ftzpmm]) + + ftzz <- mm$tz == "Z" + date[ftzz] <- as.POSIXct(date[ftzz], "UTC") + + ftz <- mm$tz != "Z" & mm$tz != "" + date[ftz] <- as.POSIXct(date[ftz], mm$tz[ftz]) + + if (default_tz != "UTC") { + ftna <- mm$tzpm == "" & mm$tz == "" + if (any(ftna)) { + dd <- as.POSIXct(format_iso_8601(date[ftna]), + "%Y-%m-%dT%H:%M:%S+00:00", tz = default_tz) + date[ftna] <- dd + } + } + + as.POSIXct(date, "UTC") +} + +iso_regex <- paste0( + "^\\s*", + "(?[\\+-]?\\d{4}(?!\\d{2}\\b))", + "(?:(?-?)", + "(?:(?0[1-9]|1[0-2])", + "(?:\\g{dash}(?[12]\\d|0[1-9]|3[01]))?", + "|W(?[0-4]\\d|5[0-3])(?:-?(?[1-7]))?", + "|(?00[1-9]|0[1-9]\\d|[12]\\d{2}|3", + "(?:[0-5]\\d|6[1-6])))", + "(?