From 35bb00095f7ebfb235424ea1ea69037519944751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Sun, 2 Apr 2023 23:29:05 +0200 Subject: [PATCH 01/46] Builder: GH releases + GHCR helper functions --- DESCRIPTION | 1 + R/builder.R | 43 +++++++ R/gh-releases.R | 205 +++++++++++++++++++++++++++++++ R/ghcr.R | 320 ++++++++++++++++++++++++++++++++++++++++++++++++ R/iso-date.R | 144 ++++++++++++++++++++++ 5 files changed, 713 insertions(+) create mode 100644 R/builder.R create mode 100644 R/gh-releases.R create mode 100644 R/ghcr.R create mode 100644 R/iso-date.R diff --git a/DESCRIPTION b/DESCRIPTION index ae793c4a..0dbc1411 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,6 +42,7 @@ Suggests: debugme, fansi, fs, + gitcreds, htmlwidgets, mockery, pak, diff --git a/R/builder.R b/R/builder.R new file mode 100644 index 00000000..9b775a71 --- /dev/null +++ b/R/builder.R @@ -0,0 +1,43 @@ + +pkg_build <- function(pkg, library = .libPaths()[1]) { + pkgdir <- file.path(library, pkg) + if (!dir.exists(pkgdir)) { + throw(pkg_error( + "Cannot find package {.pkg {pkg}} in library at {.path {library}}." + )) + } + version <- desc::desc_get_field("Version", file = pkgdir) + rversion <- get_minor_r_version(getRversion()) + platform <- pkgcache::current_r_platform() + + sys <- sysname() + if (sys == "windows") { + install_md5_sums(pkg) + fn <- paste0(pkg, "_", "R", rversion, "_", version, ".zip") + zip::zip(fn, pkgdir, mode = "cherry-pick") + + } else { + ext <- if (sys == "mac") ".tgz" else ".tar.gz" + fn <- paste0(pkg, "_", platform, "_", "R", rversion, "_", version, ext) + ffn <- file.path(normalizePath("."), fn) + old <- getwd() + on.exit(setwd(old), add = TRUE) + setwd(dirname(pkgdir)) + utils::tar(ffn, pkg, compression = "gzip", compression_level = 9) + } + + fn +} + +install_md5_sums <- function(pkgdir) { + old <- getwd() + on.exit(setwd(old), add = TRUE) + + setwd(pkgdir) + fns <- setdiff(dir(".", recursive = TRUE), "MD5") + md5 <- cli::hash_file_md5(fns) + writeLines( + paste0(md5, " *", fns), + "MD5" + ) +} diff --git a/R/gh-releases.R b/R/gh-releases.R new file mode 100644 index 00000000..7df22f9d --- /dev/null +++ b/R/gh-releases.R @@ -0,0 +1,205 @@ + +# ------------------------------------------------------------------------- + +ghr_list <- function(repo) { + synchronize(async_ghr_list(repo)) +} + +async_ghr_list <- function(repo) { + repo <- parse_slug(repo) + query <- glue("{ + rateLimit { + cost + remaining + } + repository(owner: \"\", name: \"\") { + releases(last: 100) { + nodes { + id + name + createdAt + tagName + } + } + } + }", .open = "<", .close = ">") + + github_query(query)$ + then(function(resp) { + rls <- resp$obj$data$repository$releases$nodes + browser() + data_frame( + id = vcapply(rls, "[[", "id"), + name = vcapply(rls, "[[", "name"), + tag_name = vcapply(rls, "[[", "tagName"), + created_at = parse_iso_8601(vcapply(rls, "[[", "createdAt")) + ) + }) +} + +# ------------------------------------------------------------------------- + +ghr_get <- function(repo, tag) { + synchronize(async_ghr_get(repo, tag)) +} + +async_ghr_get <- function(repo, tag) { + prepo <- parse_slug(repo) + ep <- glue("/repos/{prepo$owner}/{prepo$repo}/releases/tags/{tag}") + async_github_v3_query(ep)$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) +} + +# ------------------------------------------------------------------------- + +ghr_list_assets <- function(repo, tag) { + synchronize(async_ghr_list_assets(repo, tag)) +} + +async_ghr_list_assets <- function(repo, tag) { + + repo <- parse_slug(repo) + query <- glue("{ + rateLimit { + cost + remaining + } + repository(owner: \"\", name: \"\") { + release(tagName: \"\") { + releaseAssets(last: 100) { + nodes { + id + name + downloadUrl + size + createdAt + updatedAt + contentType + } + } + } + } + }", .open = "<", .close = ">") + + github_query(query)$ + then(function(resp) { + asts <- resp$obj$data$repository$release$releaseAssets$nodes + data_frame( + id = vcapply(asts, "[[", "id"), + name = vcapply(asts, "[[", "name"), + download_url = vcapply(asts, "[[", "downloadUrl"), + size = viapply(asts, "[[", "size"), + created_at = parse_iso_8601(vcapply(asts, "[[", "createdAt")), + updated_at = parse_iso_8601(vcapply(asts, "[[", "updatedAt")), + content_type = vcapply(asts, "[[", "contentType") + ) + }) +} + +# ------------------------------------------------------------------------- + +ghr_add_asset <- function(repo, file, tag, name = basename(file)) { + invisible(synchronize(async_ghr_add_asset(repo, file, tag, name))) +} + +async_ghr_add_asset <- function(repo, file, tag, + name = basename(file)) { + + repo; file; tag; name + + async_ghr_get(repo, tag)$ + catch(async_http_404 = function(err) { + async_ghr_create(repo, tag) + })$ + then(function(rel) rel$upload_url)$ + then(function(upload_url) { + upload_url <- sub("[{].*[}]", "", upload_url) + prepo <- parse_slug(repo) + async_github_v3_query( + url = upload_url, + endpoint = "", + query = c(name = name), + method = "POST", + headers = c("Content-Type" = "application/octet-stream"), + file = file + ) + })$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) +} + +# ------------------------------------------------------------------------- + +ghr_create <- function(repo, tag) { + invisible(synchronize(async_ghr_create(repo, tag))) +} + +async_ghr_create <- function(repo, tag, description = "", draft = FALSE, + prerelease = FALSE, + generate_release_notes = FALSE) { + prepo <- parse_slug(repo) + ep <- glue("/repos/{prepo$owner}/{prepo$repo}/releases") + data <- toJSON(list( + tag_name = tag, + name = paste0(prepo$repo, " ", tag), + body = description, + draft = draft, + prerelease = prerelease, + generate_release_notes = generate_release_notes + ), auto_unbox = TRUE) + + async_github_v3_query( + ep, + data = data, + method = "POST" + )$ + then(function(resp) { + jsonlite::fromJSON(rawToChar(resp$content), simplifyVector = FALSE) + }) +} + +# ------------------------------------------------------------------------- + +parse_slug <- function(slug) { + parts <- strsplit(slug, "/", fixed = TRUE)[[1]] + list(user = parts[1], owner = parts[1], repo = parts[2]) +} + +async_github_v3_query <- function(endpoint, query = NULL, + method = c("GET", "POST"), + headers = NULL, + data = NULL, + file = NULL, + url = NULL) { + method <- match.arg(method) + + headers <- update_named_vector(type_github_get_headers(), headers) + + base <- url %||% Sys.getenv("R_PKG_GITHUB_API_URL", "https://api.github.com") + query_str <- paste( + glue("{curl::curl_escape(names(query))}={curl::curl_escape(query)}"), + collapse = "&" + ) + url <- paste0(base, endpoint, "?", query_str) + + px <- if (method == "GET") { + http_get(url, headers = headers) + + } else if (method == "POST") { + if (is.null(data) + is.null(file) != 1) { + throw(pkg_error( + "Must specify exactly of {.arg data} and {.arg file} for POST requests." + )) + } + if (is.null(data)) data <- readBin(file, "raw", file.size(file)) + http_post(url, data, headers = headers) + } + + px$ + then(http_stop_for_status) +} + +# github_query() is in type-github.R diff --git a/R/ghcr.R b/R/ghcr.R new file mode 100644 index 00000000..98c6cfce --- /dev/null +++ b/R/ghcr.R @@ -0,0 +1,320 @@ + +ghcr_const_annotations <- list( + com.github.package.type = "r_package" +# org.opencontainers.image.authors = "Gabor Csardi", +# org.opencontainers.image.url = "https://github.com/r-lib/pak", +# org.opencontainers.image.documentation = "https://pak.r-lib.org/", +# org.opencontainers.image.source = "https://github.com/r-lib/pak", +# org.opencontainers.image.title = "pak R package", +# org.opencontainers.image.description = "Package manager for R", +# org.opencontainers.image.licenses = "GPL-3" +) + +ghcr_const_annotations_js <- function() { + paste0( + glue::glue('"{names(ghcr_const_annotations)}": "{ghcr_const_annotations}"'), + collapse = ",\n " + ) +} + +ghcr_get_package_data <- function(path) { + if (!file.exists(path)) { + throw(pkg_error("{.path {path}} does not exist.")) + } + + plt <- pkgcache::current_r_platform_data() + if (plt$os != "linux" && !grepl("^linux-", plt$os)) { + throw(pkg_error( + "{.fn ghcr_push_package} only works on Linux.", + "i" = "It does not work on this platform ({.var {plt$platform}}) yet." + )) + } + + chain_error( + dsc <- desc::desc(path), + pkg_error( + "Cannot read {.code DESCRIPTION} from {.file {path}}.", + "i" = "A valid R package must have a {.code DESCRIPTION} file." + ) + ) + + chain_error( + built <- dsc$get_built(), + pkg_error( + "Cannot find {.code Built} field in package at {.file {path}}", + "i" = "A binary R package must have a {.code Built} field in + {.code DESCRIPTION}." + ) + ) + + sha256 <- cli::hash_file_sha256(path) + data_frame( + path = path, + md5 = cli::hash_file_md5(path), + sha256 = sha256, + digest = paste0("sha256:", sha256), + image_title = paste("R package", dsc$get_field("Package")), + size = file.size(path), + buildtime = format_iso_8601(built$Date), + r_version = as.character(built$R), + r_platform = built$Platform, + package_name = dsc$get_field("Package"), + package_version = dsc$get_field("Version"), + license = dsc$get_field("License"), + arch = ghcr_canonize_arch(built$Platform), + os = ghcr_canonize_os(built$Platform) + ) +} + +ghcr_create_oci_repo <- function(pkgs, repo = NULL, oci_path = NULL, + cleanup = TRUE) { + + if (nrow(pkgs) > 1) { + throw(pkg_error("Multiple package versions are not supported yet.")) + } + + repo <- repo %||% Sys.getenv("GITHUB_REPOSITORY", NA_character_) + if (is.na(repo)) { + throw(pkg_error("Cannot detect GitHub repository")) + } + + oci_path <- oci_path %||% tempfile() + mkdirp(oci_path) + blob_path <- file.path(oci_path, "blobs", "sha256") + mkdirp(blob_path) + + # Create oci-layout file + cat( + '{"imageLayoutVersion": "1.0.0"}\n', + file = file.path(oci_path, "oci-layout") + ) + + # Copy package files to blobs + file.copy(pkgs$path, file.path(blob_path, pkgs$sha256)) + + # Config is dummy now + pkgs$image_config <- "{}" + pkgs$image_config_hash <- cli::hash_sha256(pkgs$image_config) + write_files(pkgs$image_config, file.path(blob_path, pkgs$image_config_hash)) + + # Create manifests + pkgs$manifest <- ghcr_image_manifest(pkgs) + pkgs$manifest_hash <- cli::hash_sha256(pkgs$manifest) + write_files(pkgs$manifest, file.path(blob_path, pkgs$manifest_hash)) + + # Create index + imidx <- ghcr_image_index(pkgs) + imidx_hash <- cli::hash_sha256(imidx) + write_files(imidx, file.path(blob_path, imidx_hash)) + + # index.json + idxjs <- jsonlite::prettify(glue::glue( + '{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:<>", + "size": <> + } + ] + }', .open = "<<", .close = ">>" + )) + write_files(idxjs, file.path(oci_path, "index.json")) + + # policy.json + policy_file <- file.path(oci_path, "policy.json") + policy <- jsonlite::prettify( + '{ + "default": [ + { + "type": "insecureAcceptAnything" + } + ], + "transports": + { + "docker-daemon": + { + "": [{"type":"insecureAcceptAnything"}] + } + } + }') + write_files(policy, policy_file) + + oci_path +} + +ghcr_uri <- function() { + Sys.getenv( + "PAK_GHCR_URI", + "docker://ghcr.io/r-lib/pak" + ) +} + +ghcr_user <- function() { + Sys.getenv("PAK_GHCR_USER", "gaborcsardi") +} + +ghcr_token <- function() { + Sys.getenv("PAK_GHCR_TOKEN", gitcreds::gitcreds_get()$password) +} + +ghcr_push_oci_repo <- function(oci_path, ghcr_tag = NULL) { + if (!file.exists(oci_path)) { + throw(pkg_error("Could not find OCI repository at {.path {oci_path}}.")) + } + ghcr_tag <- ghcr_tag %||% "TODO" + + skopeo <- check_skopeo() + oci_path <- normalizePath(oci_path, winslash = "/") + policy_file <- file.path(oci_path, "policy.json") + args <- c( + "copy", + "--all", + "--retry-times", 20, + "--preserve-digests", + "--policy", policy_file, + paste0("--dest-creds=", ghcr_user(), ":", ghcr_token()), + paste0("oci:", oci_path), + paste0(ghcr_uri(), ":", ghcr_tag) + ) + "TODO" +} + +ghcr_canonize_arch <- function(platform) { + arch <- strsplit(platform, "-", fixed = TRUE)[[1]][1] + c("aarch64" = "arm64", "x86_64" = "amd64")[[arch]] +} + +ghcr_canonize_os <- function(platform) { + os <- strsplit(platform, "-", fixed = TRUE)[[1]][3] + if (substr(os, 1, 6) == "darwin") os <- "darwin" + if (substr(os, 1, 5) == "mingw") os <- "windows" + if (substr(os, 1, 7) == "solaris") os <- "solaris" + os +} + +ghcr_image_index <- function(pkgs) { + tmpl <- ' + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:<>", + "size": <>, + "platform": { + "architecture": "<>", + "os": "<>", + "os.version": "R <>", + "r_version": "<>", + "r_platform": "<>" + }, + "annotations": { + "org.opencontainers.image.ref.name": "<>--<>--<>", + "io.r-hub.package.digest": "<>" + } + }' + mnfts <- glue::glue_data(pkgs, tmpl, .open = "<<", .close = ">>") + + buildtime <- max(pkgs$buildtime) + package_name <- unique(package_name) + if (length(package_name) != 1) { + throw(pkg_error( + "Cannot push different packages at the same time.", + "i" = "Found packages {.pkg {package_name}}." + )) + } + + package_version <- unique(pkgs$package_version) + if (length(package_version) != 1) { + throw(pkg_error( + "Non-matching package versions when pushing package {.pkg {package_name}}.", + "i" = "All package versions must be the same.", + "i" = "Found package versions: {.var {package_version}}." + )) + } + + tmpl2 <- ' + { + "schemaVersion": 2, + "manifests": [ + <> + ], + "annotations": { + <>, + "org.opencontainers.image.created": "<>", + "org.opencontainers.image.version": "<>" + } + }' + + jsonlite::prettify(glue::glue(tmpl2, .open = "<<", .close = ">>")) +} + +ghcr_image_manifest <- function(pkgs) { + tmpl <- ' + { + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:<>", + "size": <> + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "<>", + "size": <>, + "annotations": { + "org.opencontainers.image.title": "<>" + } + } + ], + "annotations": { + <>, + "org.opencontainers.image.created": "<>", + "org.opencontainers.image.version": "<>" + } + }' + + glue::glue_data(pkgs, tmpl, .open = "<<", .close = ">>") +} + + +write_file <- function(txt, path) { + out <- file(path, open = "wb") + on.exit(close(out), add = TRUE) + cat(txt, file = out, sep = "") +} + +write_files <- function(txts, paths) { + invisible(mapply(write_file, txts, paths)) +} + +find_skopeo <- function() { + path <- Sys.which("skopeo") + if (path != "") return(path) + if (file.exists(cand <- "/usr/local/bin/skopeo")) return(cand) + if (file.exists(cand <- "/opt/homebrew/bin/skopeo")) return(cand) + throw(pkg_error("Need skopeo to push packages.")) +} + +skopeo_version <- function() { + skopeo <- find_skopeo() + out <- processx::run(skopeo, "--version") + re_ver <- "[ ]([0-9]+[.][0-9]+[.][0-9]+)" + if (!grepl(re_ver, out$stdout)) stop("Cannot determine skopeo version") + mch <- regexpr(re_ver, out$stdout, perl = TRUE) + beg <- attr(mch, "capture.start")[1] + end <- beg + attr(mch, "capture.length")[1] - 1L + package_version(substr(out$stdout, beg, end)) +} + +check_skopeo <- function() { + skopeo <- find_skopeo() + skopeo_ver <- skopeo_version() + if (skopeo_ver < "1.6.0") { + throw(pkg_error( + "Need at least skopeo 1.6.0 to push packages", + "i" = "Found skopeo {.emph {skopoe_ver}} at {.path {skopeo}}." + )) + } + skopeo[[1]] +} diff --git a/R/iso-date.R b/R/iso-date.R new file mode 100644 index 00000000..687af22a --- /dev/null +++ b/R/iso-date.R @@ -0,0 +1,144 @@ + +milliseconds <- function(x) as.difftime(as.numeric(x) / 1000, units = "secs") +seconds <- function(x) as.difftime(as.numeric(x), units = "secs") +minutes <- function(x) as.difftime(as.numeric(x), units = "mins") +hours <- function(x) as.difftime(as.numeric(x), units = "hours") +days <- function(x) as.difftime(as.numeric(x), units = "days") +weeks <- function(x) as.difftime(as.numeric(x), units = "weeks") +wday <- function(x) as.POSIXlt(x, tz = "UTC")$wday + 1 +with_tz <- function(x, tzone = "") as.POSIXct(as.POSIXlt(x, tz = tzone)) +ymd <- function(x) as.POSIXct(x, format = "%Y %m %d", tz = "UTC") +yj <- function(x) as.POSIXct(x, format = "%Y %j", tz = "UTC") + +parse_iso_8601 <- function(dates, default_tz = "UTC") { + if (default_tz == "") default_tz <- Sys.timezone() + dates <- as.character(dates) + match <- re_match(dates, iso_regex) + matching <- !is.na(match$.match) + result <- rep(.POSIXct(NA_real_, tz = ""), length.out = length(dates)) + result[matching] <- parse_iso_parts(match[matching, ], default_tz) + class(result) <- c("POSIXct", "POSIXt") + with_tz(result, "UTC") +} + +parse_iso_parts <- function(mm, default_tz) { + + num <- nrow(mm) + + ## ----------------------------------------------------------------- + ## Date first + + date <- .POSIXct(rep(NA_real_, num), tz = "") + + ## Years-days + fyd <- is.na(date) & mm$yearday != "" + date[fyd] <- yj(paste(mm$year[fyd], mm$yearday[fyd])) + + ## Years-weeks-days + fywd <- is.na(date) & mm$week != "" & mm$weekday != "" + date[fywd] <- iso_week(mm$year[fywd], mm$week[fywd], mm$weekday[fywd]) + + ## Years-weeks + fyw <- is.na(date) & mm$week != "" + date[fyw] <- iso_week(mm$year[fyw], mm$week[fyw], "1") + + ## Years-months-days + fymd <- is.na(date) & mm$month != "" & mm$day != "" + date[fymd] <- ymd(paste(mm$year[fymd], mm$month[fymd], mm$day[fymd])) + + ## Years-months + fym <- is.na(date) & mm$month != "" + date[fym] <- ymd(paste(mm$year[fym], mm$month[fym], "01")) + + ## Years + fy <- is.na(date) + date[fy] <- ymd(paste(mm$year, "01", "01")) + + ## ----------------------------------------------------------------- + ## Now the time + + th <- mm$hour != "" + date[th] <- date[th] + hours(mm$hour[th]) + + tm <- mm$min != "" + date[tm] <- date[tm] + minutes(mm$min[tm]) + + ts <- mm$sec != "" + date[ts] <- date[ts] + seconds(mm$sec[ts]) + + ## ----------------------------------------------------------------- + ## Fractional time + + frac <- as.numeric(sub(",", ".", mm$frac)) + + tfs <- !is.na(frac) & mm$sec != "" + date[tfs] <- date[tfs] + milliseconds(round(frac[tfs] * 1000)) + + tfm <- !is.na(frac) & mm$sec == "" & mm$min != "" + sec <- trunc(frac[tfm] * 60) + mil <- round((frac[tfm] * 60 - sec) * 1000) + date[tfm] <- date[tfm] + seconds(sec) + milliseconds(mil) + + tfh <- !is.na(frac) & mm$sec == "" & mm$min == "" + min <- trunc(frac[tfh] * 60) + sec <- trunc((frac[tfh] * 60 - min) * 60) + mil <- round((((frac[tfh] * 60) - min) * 60 - sec) * 1000) + date[tfh] <- date[tfh] + minutes(min) + seconds(sec) + milliseconds(mil) + + ## ----------------------------------------------------------------- + ## Time zone + + ftzpm <- mm$tzpm != "" + m <- ifelse(mm$tzpm[ftzpm] == "+", -1, 1) + ftzpmh <- ftzpm & mm$tzhour != "" + date[ftzpmh] <- date[ftzpmh] + m * hours(mm$tzhour[ftzpmh]) + ftzpmm <- ftzpm & mm$tzmin != "" + date[ftzpmm] <- date[ftzpmm] + m * minutes(mm$tzmin[ftzpmm]) + + ftzz <- mm$tz == "Z" + date[ftzz] <- as.POSIXct(date[ftzz], "UTC") + + ftz <- mm$tz != "Z" & mm$tz != "" + date[ftz] <- as.POSIXct(date[ftz], mm$tz[ftz]) + + if (default_tz != "UTC") { + ftna <- mm$tzpm == "" & mm$tz == "" + if (any(ftna)) { + dd <- as.POSIXct(format_iso_8601(date[ftna]), + "%Y-%m-%dT%H:%M:%S+00:00", tz = default_tz) + date[ftna] <- dd + } + } + + as.POSIXct(date, "UTC") +} + +iso_regex <- paste0( + "^\\s*", + "(?[\\+-]?\\d{4}(?!\\d{2}\\b))", + "(?:(?-?)", + "(?:(?0[1-9]|1[0-2])", + "(?:\\g{dash}(?[12]\\d|0[1-9]|3[01]))?", + "|W(?[0-4]\\d|5[0-3])(?:-?(?[1-7]))?", + "|(?00[1-9]|0[1-9]\\d|[12]\\d{2}|3", + "(?:[0-5]\\d|6[1-6])))", + "(?