From dcba62cc2e85d88137cb18d827a21aae9a0bacbf Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:04:40 -0400 Subject: [PATCH 01/29] add parse repo helper and check function --- R/utils.R | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/R/utils.R b/R/utils.R index 60853a1..37ef0fd 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,6 +1,18 @@ +#' Parses repository spec and errors if it fails +#' @keywords internal +#' @noRd +parse_repo <- function(repo){ + r <- strsplit(repo, "/")[[1]] + + if (length(r) != 2) { + cli::cli_abort( + c("Could not parse {.val {repo}} as a GitHub repository.", + "Make sure you have used the format: {.val owner/repo}") + ) + } - - + return(r) +} # Adapted from `usethis` and under # GPL-3 (Copyright RStudio Inc) @@ -67,7 +79,6 @@ cat_line <- function(...) { compact <- function (l) Filter(Negate(is.null), l) - # utils::askYesKnow is new to R 3.5.0; avoid using it for backwards compatibility askYesNo <- function(msg){ From 5ca1f515df48d64300ac830566850557c196260d Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:07:48 -0400 Subject: [PATCH 02/29] Create pb_releases() and refactor pb_info() pb_releases() ONLY returns a list of the releases available in repo. pb_info now checks for tag specifications and ONLY returns assets for the specified tags (if specified). Also pb_info no longer asks about creating new releases, nor does it warn about missing releases. also, memoise should happen in .onLoad() --- R/pb_info.R | 292 ++++++++++++++++++++++++---------------------------- 1 file changed, 135 insertions(+), 157 deletions(-) diff --git a/R/pb_info.R b/R/pb_info.R index 004e255..905cbbb 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -1,186 +1,164 @@ -release_info <- function(repo = guess_repo(), .token = get_token()) { - r <- strsplit(repo, "/")[[1]] - if (length(r) != 2) { - stop(paste( - "Could not parse", r, "as a repository", - "Make sure you have used the format:", - crayon::blue$bold("owner/repo") - )) - } +#' List releases in repository +#' +#' +#' @param repo GitHub repository specification in the form of `"owner/repo"`, if not specified will try to guess repo based on current working directory. +#' @param .token a GitHub API token, defaults to `gh::gh_token()` +#' +#' @examples +#' \donttest{ +#' try({ # wrapped in try block to prevent CRAN errors +#' pb_releases("nflverse/nflverse-data") +#' }) +#' } +#' @return a dataframe of all releases available within a repository. +#' +#' @export +pb_releases <- function(repo = guess_repo(), .token = gh::gh_token()){ + + r <- parse_repo(repo) # get release ids - releases <- maybe(gh::gh("/repos/:owner/:repo/releases", - owner = r[[1]], repo = r[[2]], - .limit = Inf, - .token = .token - ), - otherwise = stop(api_error_msg(r)) + releases <- maybe( + expr = gh::gh("/repos/:owner/:repo/releases", + owner = r[[1]], + repo = r[[2]], + .limit = Inf, + .token = .token), + otherwise = stop(api_error_msg(r)) ) - # if there are no releases, don't update assets - if (length(releases) == 1 && identical(releases[[1]], "")) return(releases) - # fetch asset meta-data individually, see #19 - for (i in seq_along(releases)) { - a <- gh::gh(endpoint = "/repos/:owner/:repo/releases/:release_id/assets", - owner = r[[1]], - repo = r[[2]], - release_id = releases[[i]]$id, - .limit = Inf, - .token = .token) - if(length(a) == 0) next - if (!identical(a[[1]], "")) { - # if the i'th release does not have any assets then we skip updating - # the assets in the releases object - ## Now use assets given by the release id as the returned assets - class(a) <- "list" - attributes(a) <- NULL - releases[[i]]$assets <- a - } - } - - # return result - releases -} - - - -#' @importFrom lubridate as_datetime -release_data <- function(x, r) { - - if(!"assets" %in% names(x)) + if(length(releases) == 0) { + cli::cli_warn( + c("!" = "No GitHub releases found for {.val {repo}}!", + "You can make a new one with {.fun piggyback::pb_new_release}") + ) return(data.frame()) - - if(length(x$assets) == 0){ - ## Release exists but has no assets - return( - data.frame( - file_name = "", - size = 0L, - timestamp = lubridate::as_datetime(0), - tag = x$tag_name, - owner = r[[1]], - repo = r[[2]], - upload_url = x$upload_url, - browser_download_url = "", - id = "", - state = "", - stringsAsFactors = FALSE - )) - } - data.frame( - file_name = local_filename( - vapply(x$assets, `[[`, character(1), "name") - ), - size = - vapply(x$assets, `[[`, integer(1), "size"), - timestamp = lubridate::as_datetime( - vapply(x$assets, `[[`, character(1), "updated_at") - ), - tag = x$tag_name, - owner = r[[1]], - repo = r[[2]], - upload_url = x$upload_url, - browser_download_url = vapply( - x$assets, `[[`, character(1L), - "browser_download_url" - ), - id = vapply(x$assets, `[[`, integer(1L), "id"), - state = vapply(x$assets, `[[`, character(1L), "state"), - stringsAsFactors = FALSE + out <- data.frame( + release_name = vapply(releases, `[[`, character(1),"name"), + release_id = vapply(releases, `[[`, integer(1),"id"), + release_body = vapply(releases, `[[`, character(1),"body"), + tag_name = vapply(releases, `[[`, character(1),"tag_name"), + draft = vapply(releases, `[[`, logical(1),"draft"), + created_at = vapply(releases, `[[`, character(1),"created_at"), + published_at = vapply(releases, `[[`, character(1),"published_at"), + html_url = vapply(releases, `[[`, character(1),"html_url"), + upload_url = vapply(releases, `[[`,character(1),"upload_url"), + n_assets = vapply(releases, function(x) length(x[["assets"]]), integer(1)) ) -} -null_chr <- function(x){ - if(is.null(x)) return("") - as.character(NA) + return(out) } -pb_info_fn <- function(repo = guess_repo(), - tag = NULL, - .token = get_token()) { - - releases <- release_info(repo, .token) - ## FIXME Establish if any releases exist first! - if(length(releases) == 0){ - if(!interactive()){ - message("no releases found") - return(data.frame()) - } else { - continue <- askYesNo(paste("No releases found", - "would you like to create one?")) - if(continue){ - pb_new_release(repo=repo, tag=tag, .token = .token) - - } else { - return(data.frame()) - } - } +#' @keywords internal +#' @param releases as created by `pb_releases()` +#' @param r a list of owner/repo as created by `parse_repo()` +#' @noRd +get_release_assets <- function(releases, r) { - } + if(nrow(releases)==0) return(data.frame()) + + asset_list <- vector("list", length = nrow(releases)) - r <- strsplit(repo, "/")[[1]] - - info <- do.call(rbind, lapply(releases, release_data, r)) - - if (!is.null(tag)) { - if (tag == "latest") { - info <- info[info$tag == info$tag[[1]], ] - } else if (tag %in% info$tag) { - info <- info[info$tag == tag, ] - } else { - if (!interactive()) { - stop(paste0( - "No release with tag ", tag, " exists on repo ", repo, - ". You can create a new release with pb_new_release() function." - )) - } else { - create <- askYesNo(paste( - "release tag", tag, - "does not exist. Would you like to create it?" - )) - if (create) { - pb_new_release(repo, tag, .token = .token) - # Recursion! - info <- pb_info_fn(repo, tag,.token) - } else { - return(NULL) - } - } + # fetch asset meta-data individually, see #19 + for (i in seq_along(releases$tag_name)) { + a <- gh::gh(endpoint = "/repos/:owner/:repo/releases/:release_id/assets", + owner = r[[1]], + repo = r[[2]], + release_id = releases$release_id[[i]], + .limit = Inf, + .token = .token) + if(length(a) == 0) next + if (!identical(a[[1]], "")) { + + # convert list to dataframe and store in asset list + + a_df <- data.frame( + file_name = vapply(a, `[[`, character(1), "name"), + size = vapply(a, `[[`, integer(1), "size"), + timestamp = lubridate::as_datetime(vapply(a, `[[`, character(1), "updated_at")), + tag = releases$tag_name[i], + owner = r[[1]], + repo = r[[2]], + upload_url = releases$upload_url[i], + browser_download_url = vapply(a, `[[`, character(1L),"browser_download_url"), + id = vapply(a, `[[`, integer(1L), "id"), + state = vapply(a, `[[`, character(1L), "state"), + stringsAsFactors = FALSE + ) + + asset_list[[i]] <- a_df } } - # we deleted a file, so we better break cache. - memoise::forget(memoised_pb_info) + # convert list of asset dataframes to single dataframe + release_assets <- do.call(rbind,asset_list) - info + # return result + return(release_assets) } -memoised_pb_info <- - memoise::memoise(pb_info_fn, - ~memoise::timeout(as.numeric( - Sys.getenv("piggyback_cache_duration", "600") - )) - ) - - -#' @importFrom memoise memoise timeout forget pb_info <- function(repo = guess_repo(), tag = NULL, - .token = get_token(), - cache = TRUE){ + .token = gh::gh_token()) { - seconds <- as.numeric(Sys.getenv("piggyback_cache_duration", "600")) - if(seconds == 0) cache <- FALSE - if(cache){ + r <- parse_repo(repo) - memoised_pb_info(repo, tag, .token) + # get all releases + releases <- pb_releases(repo, .token) - } else { - pb_info_fn(repo, tag, .token) + # if no releases return empty df + if(nrow(releases) == 0) { + return( + data.frame( + file_name = "", + size = 0L, + timestamp = lubridate::as_datetime(0), + tag = x$tag_name, + owner = r[[1]], + repo = r[[2]], + upload_url = x$upload_url, + browser_download_url = "", + id = "", + state = "", + stringsAsFactors = FALSE + )) } + # if tag is latest, do call for first tag only + if(!is.null(tag) && length(tag) == 1 && tag == "latest") tag <- releases$tag_name[[1]] + + # if tag is present, filter the releases to search to just the tags requested + if(!is.null(tag)) releases <- releases[releases$tag_name %in% tag,] + + # get release assets and metadata for each release + info <- get_release_assets(releases, r) + + return(info) } +# memoised_pb_info <- +# memoise::memoise(pb_info_fn, +# ~memoise::timeout(as.numeric( +# Sys.getenv("piggyback_cache_duration", "600") +# )) +# ) + +# pb_info <- function(repo = guess_repo(), +# tag = NULL, +# .token = get_token(), +# cache = TRUE){ +# +# seconds <- as.numeric(Sys.getenv("piggyback_cache_duration", "600")) +# if(seconds == 0) cache <- FALSE +# if(cache){ +# memoised_pb_info(repo, tag, .token) +# } else { +# pb_info_fn(repo, tag, .token) +# } +# +# } + + From c241495c4b093195352f70c6fa64a3fd260705f6 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:39:28 -0400 Subject: [PATCH 03/29] add memoising of pb_info on load --- R/zzz.R | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 R/zzz.R diff --git a/R/zzz.R b/R/zzz.R new file mode 100644 index 0000000..4acb9d1 --- /dev/null +++ b/R/zzz.R @@ -0,0 +1,6 @@ +.onLoad <- function(libname,pkgname){ + cache_duration <- as.numeric(Sys.getenv("piggyback_cache_duration", "600")) + assign(x = "pb_info", + value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), + envir = parent.env(environment())) +} From b061e079ebedd315ffd41de6084634477a2f19d2 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:55:23 -0400 Subject: [PATCH 04/29] pb_upload fixes Add ask to create new release and rewrite messaging with {cli} --- R/pb_upload.R | 66 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/R/pb_upload.R b/R/pb_upload.R index 7b5aaa8..0fa6557 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -22,7 +22,6 @@ #' readr::write_tsv(mtcars,"mtcars.tsv.xz") #' pb_upload("mtcars.tsv.xz", "cboettig/piggyback-tests") #' } -#' @importFrom httr progress upload_file POST stop_for_status #' @export #' pb_upload <- function(file, @@ -32,11 +31,38 @@ pb_upload <- function(file, overwrite = "use_timestamps", use_timestamps = NULL, show_progress = TRUE, - .token = get_token(), + .token = gh::gh_token(), dir = NULL) { + stopifnot( + is.character(repo), + is.character(tag), + length(tag) == 1, + length(repo) == 1 + ) + + releases <- pb_releases(repo, .token) + + if(tag != "latest" && !tag %in% releases$tag_name) { + cli::cli_alert_warning("Release {.val {tag}} not found in {.val {repo}}.") + + if(!interactive()) { + cli::cli_alert_warning("No upload performed.") + return(invisible(NULL)) + } + + run <- utils::menu( + choices = c("Yes", "No"), + title = glue::glue("Would you like to create a new release now?") + ) + + if(run == "No") return(invisible(NULL)) + + if(run == "Yes") pb_new_release(repo = repo, tag = tag, .token = token) + } + ## start fresh - memoise::forget(memoised_pb_info) + memoise::forget(pb_info) out <- lapply(file, function(f) pb_upload_file( @@ -52,7 +78,7 @@ pb_upload <- function(file, )) ## break cache when done - memoise::forget(memoised_pb_info) + memoise::forget(pb_info) invisible(out) } @@ -75,17 +101,14 @@ pb_upload_file <- function(file, ## return "./C:/Users/Tan" which is not desired. if (!file.exists(file_path)) { - warning("file ", file_path, " does not exist") + cli::cli_warn("file {.file {file_path}} does not exist.") return(NULL) } if(!is.null(use_timestamps)){ - warning(paste("use_timestamps argument is deprecated", - "please set overwrite='use_timestamps'", - "instead.")) + cli::cli_warn("{.code use_timestamps} argument is deprecated, please set {.code overwrite = 'use_timestamps'} instead") } - ## Yeah, having two separate arguments was clearly a mistake! ## Code has been partially refactored now so that user just ## sets `overwrite` and we handle the twisted logic internally here: @@ -122,10 +145,7 @@ pb_upload_file <- function(file, no_update <- local_timestamp <= df[i, "timestamp"] if (no_update) { - message(paste( - "matching or more recent version of", - file_path, "found on GitHub, not uploading" - )) + cli::cli_warn("Matching or more recent version of {.file {file_path}} found on GH, not uploading.") return(NULL) } } @@ -139,30 +159,24 @@ pb_upload_file <- function(file, .token = .token ) } else { - warning(paste( - "Skipping upload of", df$file_name[i], - "as file exists on GitHub", - repo, "and overwrite = FALSE" - )) + cli::cli_warn("Skipping upload of {.file {df$file_name[i]}} as file exists on GitHub and {.code overwrite = FALSE}") return(NULL) } } - if (!is.null(progress)) { - message(paste("uploading", name, "...")) - } + if (!is.null(progress)) cli::cli_alert_info("Uploading {.file {name}} ...") r <- httr::POST(sub("\\{.+$", "", df$upload_url[[1]]), - query = list(name = asset_filename(name)), + query = list(name = name), httr::add_headers(Authorization = paste("token", .token)), body = httr::upload_file(file_path), - progress - ) + progress) cat("\n") - if(getOption("verbose")) httr::warn_for_status(r) + + if(getOption("piggyback.verbose")) httr::warn_for_status(r) ## Release info changed, so break cache - # memoise::forget(memoised_pb_info) + memoise::forget(pb_info) invisible(r) } From 6b2752f005ade8bb04adee1090bf12f25d29ab97 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:56:21 -0400 Subject: [PATCH 05/29] redocument --- man/pb_delete.Rd | 2 +- man/pb_list.Rd | 20 +++----------------- man/pb_releases.Rd | 26 ++++++++++++++++++++++++++ man/piggyback-package.Rd | 1 + 4 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 man/pb_releases.Rd diff --git a/man/pb_delete.Rd b/man/pb_delete.Rd index 38251a2..99d0955 100644 --- a/man/pb_delete.Rd +++ b/man/pb_delete.Rd @@ -8,7 +8,7 @@ pb_delete( file = NULL, repo = guess_repo(), tag = "latest", - .token = get_token() + .token = gh::gh_token() ) } \arguments{ diff --git a/man/pb_list.Rd b/man/pb_list.Rd index 963287f..04eaa5d 100644 --- a/man/pb_list.Rd +++ b/man/pb_list.Rd @@ -4,37 +4,23 @@ \alias{pb_list} \title{List all assets attached to a release} \usage{ -pb_list( - repo = guess_repo(), - tag = NULL, - ignore = "manifest.json", - .token = get_token() -) +pb_list(repo = guess_repo(), tag = NULL, .token = get_token()) } \arguments{ \item{repo}{Repository name in format "owner/repo". Will guess the current repo if not specified.} -\item{tag}{which release tag do we want information for? If \code{NULL} (default), +\item{tag}{which release tag(s) do we want information for? If \code{NULL} (default), will return a table for all available release tags.} -\item{ignore}{a list of files to ignore (if downloading "all" because -\code{file=NULL}).} - \item{.token}{GitHub authentication token, see \verb{[gh::gh_token()]}} } \value{ -a data.frame of release asset names, (normalized to local paths), release tag, -timestamp, owner, and repo. +a data.frame of release asset names, release tag, timestamp, owner, and repo. } \description{ List all assets attached to a release } -\details{ -To preserve path information, local path delimiters are converted to \verb{.2f} -when files are uploaded as assets. Listing will display the local filename, -with asset names converting the \verb{.2f} escape code back to the system delimiter. -} \examples{ \dontrun{ pb_list("cboettig/piggyback-tests") diff --git a/man/pb_releases.Rd b/man/pb_releases.Rd new file mode 100644 index 0000000..c1b5213 --- /dev/null +++ b/man/pb_releases.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/pb_info.R +\name{pb_releases} +\alias{pb_releases} +\title{List releases in repository} +\usage{ +pb_releases(repo = guess_repo(), .token = gh::gh_token()) +} +\arguments{ +\item{repo}{GitHub repository specification in the form of \code{"owner/repo"}, if not specified will try to guess repo based on current working directory.} + +\item{.token}{a GitHub API token, defaults to \code{gh::gh_token()}} +} +\value{ +a dataframe of all releases available within a repository. +} +\description{ +List releases in repository +} +\examples{ +\donttest{ +try({ # wrapped in try block to prevent CRAN errors + pb_releases("nflverse/nflverse-data") +}) +} +} diff --git a/man/piggyback-package.Rd b/man/piggyback-package.Rd index f8fc64f..22088a7 100644 --- a/man/piggyback-package.Rd +++ b/man/piggyback-package.Rd @@ -40,6 +40,7 @@ Other contributors: \item Mark Padgham (\href{https://orcid.org/0000-0003-2172-5265}{ORCID}) [contributor] \item Jeffrey O Hanson (\href{https://orcid.org/0000-0002-4716-6134}{ORCID}) [contributor] \item Kevin Kuo (\href{https://orcid.org/0000-0001-7803-7901}{ORCID}) [contributor] + \item Tan Ho (\href{https://orcid.org/0000-0001-8388-5155}{ORCID}) [contributor] } } From 40108d3e133bb1a1d7b6681da36060c3a47480a6 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:57:06 -0400 Subject: [PATCH 06/29] remove build-time memoise --- R/pb_info.R | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/R/pb_info.R b/R/pb_info.R index 905cbbb..c9bdfd6 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -138,27 +138,3 @@ pb_info <- function(repo = guess_repo(), return(info) } -# memoised_pb_info <- -# memoise::memoise(pb_info_fn, -# ~memoise::timeout(as.numeric( -# Sys.getenv("piggyback_cache_duration", "600") -# )) -# ) - -# pb_info <- function(repo = guess_repo(), -# tag = NULL, -# .token = get_token(), -# cache = TRUE){ -# -# seconds <- as.numeric(Sys.getenv("piggyback_cache_duration", "600")) -# if(seconds == 0) cache <- FALSE -# if(cache){ -# memoised_pb_info(repo, tag, .token) -# } else { -# pb_info_fn(repo, tag, .token) -# } -# -# } - - - From 96c934e6cc0a6cdc453904f124634c2c5d5216ac Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:57:38 -0400 Subject: [PATCH 07/29] update pb_delete with new cli and fix checks --- R/pb_delete.R | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/R/pb_delete.R b/R/pb_delete.R index 3a3e7e5..240f34b 100644 --- a/R/pb_delete.R +++ b/R/pb_delete.R @@ -22,25 +22,37 @@ pb_delete <- function(file = NULL, repo = guess_repo(), tag = "latest", - .token = get_token()) { + .token = gh::gh_token()) { df <- pb_info(repo, tag, .token) - if (is.null(file)) { - ids <- df$id - } else { + if (is.null(file)) ids <- df$id + + if(!is.null(file) && any(!file %in% df$file_name)){ + missing <- file[!file %in% df$file_name] + file <- file[file != missing] + cli::cli_warn("{.val {missing}} not found in {.val {tag}} release of {.val {repo}}") + } + + if(!is.null(file)){ ids <- df[df$file_name %in% file, "id"] } if (length(ids) < 1) { - message(paste(file, "not found on GitHub")) - return(NULL) + cli::cli_warn("No file deletions performed.") + return(invisible(NULL)) } lapply(ids, function(id) { ## If we find matching id, Delete file from release. gh::gh("DELETE /repos/:owner/:repo/releases/assets/:id", - owner = df$owner[[1]], repo = df$repo[[1]], id = id, .token = .token + owner = df$owner[[1]], + repo = df$repo[[1]], + id = id, + .token = .token ) }) - invisible(TRUE) + + cli::cli_alert_info("Deleted {.val {file}} from {.val {tag}} release on {.val {repo}}") + + return(invisible(TRUE)) } From 9da26eae330f9f64bd436b8f1c7e65d20db863b9 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:58:01 -0400 Subject: [PATCH 08/29] update docs --- R/pb_list.R | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/R/pb_list.R b/R/pb_list.R index 469d908..ba4b392 100644 --- a/R/pb_list.R +++ b/R/pb_list.R @@ -1,21 +1,16 @@ - #' List all assets attached to a release #' @inheritParams pb_download -#' @param tag which release tag do we want information for? If `NULL` (default), +#' @param tag which release tag(s) do we want information for? If `NULL` (default), #' will return a table for all available release tags. -#' @return a data.frame of release asset names, (normalized to local paths), release tag, -#' timestamp, owner, and repo. -#' @details To preserve path information, local path delimiters are converted to `.2f` -#' when files are uploaded as assets. Listing will display the local filename, -#' with asset names converting the `.2f` escape code back to the system delimiter. +#' @return a data.frame of release asset names, release tag, timestamp, owner, and repo. #' @examples #' \dontrun{ #' pb_list("cboettig/piggyback-tests") #' } +#' @seealso `pb_releases` for a list of all releases in repository #' @export pb_list <- function(repo = guess_repo(), tag = NULL, - ignore = "manifest.json", .token = get_token()) { df <- pb_info(repo, tag, .token) df[c("file_name", "size", "timestamp", "tag", "owner", "repo")] From 4c3516481cf7d585901b1f2fc5f07abd1be2fe9e Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Sun, 24 Apr 2022 23:59:41 -0400 Subject: [PATCH 09/29] switch out get_token for gh_token --- NAMESPACE | 5 +---- R/gh.R | 9 --------- R/pb_download.R | 4 ++-- R/pb_download_url.R | 2 +- R/pb_list.R | 2 +- R/pb_new_release.R | 24 +++++++++--------------- R/pb_upload.R | 2 +- tests/testthat/test-errors.R | 2 +- tests/testthat/test-timestamps.R | 4 ++-- tests/testthat/test-with-auth.R | 12 ++++++------ tests/testthat/test-without-auth.R | 2 +- 11 files changed, 25 insertions(+), 43 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index eb20c49..941e476 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,6 +5,7 @@ export(pb_download) export(pb_download_url) export(pb_list) export(pb_new_release) +export(pb_releases) export(pb_upload) importFrom(clisymbols,symbol) importFrom(crayon,blue) @@ -21,7 +22,3 @@ importFrom(httr,stop_for_status) importFrom(httr,upload_file) importFrom(httr,write_disk) importFrom(jsonlite,toJSON) -importFrom(lubridate,as_datetime) -importFrom(memoise,forget) -importFrom(memoise,memoise) -importFrom(memoise,timeout) diff --git a/R/gh.R b/R/gh.R index 8697c4a..ee3642d 100644 --- a/R/gh.R +++ b/R/gh.R @@ -23,17 +23,8 @@ api_error_msg <- function(r) { ) } - - - - -get_token <- function(warn = TRUE) { - gh::gh_token() -} ##################################################### - - maybe <- function(expr, otherwise, quiet = TRUE) { if (missing(otherwise)) { try(expr, silent = quiet) diff --git a/R/pb_download.R b/R/pb_download.R index 094319a..eeb30f2 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -35,7 +35,7 @@ pb_download <- function(file = NULL, ignore = "manifest.json", use_timestamps = TRUE, show_progress = TRUE, - .token = get_token()) { + .token = gh::gh_token()()) { progress <- httr::progress("down") if (!show_progress) { progress <- NULL @@ -111,7 +111,7 @@ gh_download_asset <- function(owner, id, destfile, overwrite = TRUE, - .token = get_token(), + .token = gh::gh_token()(), progress = httr::progress("down")) { if (fs::file_exists(destfile) && !overwrite) { warning(paste( diff --git a/R/pb_download_url.R b/R/pb_download_url.R index 32e3863..4991a0c 100644 --- a/R/pb_download_url.R +++ b/R/pb_download_url.R @@ -17,7 +17,7 @@ pb_download_url <- function(file = NULL, repo = guess_repo(), tag = "latest", - .token = get_token()) { + .token = gh::gh_token()()) { df <- pb_info(repo, tag, .token) if (is.null(file)) { return(df$browser_download_url) diff --git a/R/pb_list.R b/R/pb_list.R index ba4b392..2470d6e 100644 --- a/R/pb_list.R +++ b/R/pb_list.R @@ -11,7 +11,7 @@ #' @export pb_list <- function(repo = guess_repo(), tag = NULL, - .token = get_token()) { + .token = gh::gh_token()()) { df <- pb_info(repo, tag, .token) df[c("file_name", "size", "timestamp", "tag", "owner", "repo")] } diff --git a/R/pb_new_release.R b/R/pb_new_release.R index b672d7a..2483db0 100644 --- a/R/pb_new_release.R +++ b/R/pb_new_release.R @@ -29,22 +29,16 @@ pb_new_release <- function(repo = guess_repo(), body = "Data release", draft = FALSE, prerelease = FALSE, - .token = get_token()) { + .token = gh::gh_token()()) { - releases <- release_info(repo, .token) + releases <- pb_releases(repo, .token) - # if no releases exist, release_info returns a gh_response length-0 list - if(length(releases) > 0){ - # Otherwise, list is at least length 1, with names. - if("tag_name" %in% names(releases[[1]])){ - current_tags <- lapply(releases, `[[`, "tag_name") - if (tag %in% current_tags) { - stop(paste("release tag", tag, "already exists")) - } - } + # if no releases exist, pb_releases returns a dataframe of releases + if(nrow(releases) > 0 && tag %in% releases$tag_name){ + cli::cli_abort("Release tag {.val {tag}} already exists!") } - r <- strsplit(repo, "/")[[1]] + r <- parse_repo(repo) payload <- compact(list( tag_name = tag, @@ -55,9 +49,9 @@ pb_new_release <- function(repo = guess_repo(), prerelease = prerelease )) - ## gh fails to pass body correctly?? - #gh("/repos/:owner/:repo/releases", owner = r[[1]], repo = r[[2]], - # .method = "POST", body = toJSON(payload,auto_unbox = TRUE), encode="json") + # gh fails to pass body correctly?? + # gh("/repos/:owner/:repo/releases", owner = r[[1]], repo = r[[2]], + # .method = "POST", body = toJSON(payload,auto_unbox = TRUE), encode="json") resp <- httr::POST(paste0( "https://api.github.com/repos/", r[[1]], "/", diff --git a/R/pb_upload.R b/R/pb_upload.R index 0fa6557..41b07e5 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -89,7 +89,7 @@ pb_upload_file <- function(file, overwrite = "use_timestamps", use_timestamps = NULL, show_progress = TRUE, - .token = get_token(), + .token = gh::gh_token()(), dir = NULL) { file_path <- do.call(file.path, compact(list(dir,file))) diff --git a/tests/testthat/test-errors.R b/tests/testthat/test-errors.R index a0fd573..303f3da 100644 --- a/tests/testthat/test-errors.R +++ b/tests/testthat/test-errors.R @@ -188,7 +188,7 @@ test_that("download url error", { x <- pb_download_url("not-a-file", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::get_token() + .token = piggyback:::gh::gh_token()() ), "not-a-file") }) diff --git a/tests/testthat/test-timestamps.R b/tests/testthat/test-timestamps.R index 74de200..de74a21 100644 --- a/tests/testthat/test-timestamps.R +++ b/tests/testthat/test-timestamps.R @@ -5,8 +5,8 @@ testthat::test_that( "upload obeys timestamp only when newer", { testthat::skip_on_travis() ## No idea. - testthat::skip_if(piggyback:::get_token() == "") - testthat::skip_if(piggyback:::get_token() == + testthat::skip_if(piggyback:::gh::gh_token()() == "") + testthat::skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") testthat::skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-with-auth.R b/tests/testthat/test-with-auth.R index 1954ed6..247c1a9 100644 --- a/tests/testthat/test-with-auth.R +++ b/tests/testthat/test-with-auth.R @@ -6,7 +6,7 @@ context("Requiring Authentication") test_that("We can upload data", { # public pat - skip_if(piggyback:::get_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") data <- readr::write_tsv(datasets::iris, "iris.tsv.gz") @@ -26,7 +26,7 @@ test_that("We can upload data", { test_that("working from git repo", { # public pat - skip_if(piggyback:::get_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") ## Setup @@ -85,13 +85,13 @@ test_that("working from git repo", { test_that("we can get a download url", { # public pat - skip_if(piggyback:::get_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") x <- pb_download_url("iris.tsv.gz", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::get_token() + .token = piggyback:::gh::gh_token()() ) expect_is(x, "character") }) @@ -114,8 +114,8 @@ test_that( testthat::test_that( "test delete", { - testthat::skip_if(piggyback:::get_token() == "") - testthat::skip_if(piggyback:::get_token() == + testthat::skip_if(piggyback:::gh::gh_token()() == "") + testthat::skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-without-auth.R b/tests/testthat/test-without-auth.R index 5a932d3..6f0324f 100644 --- a/tests/testthat/test-without-auth.R +++ b/tests/testthat/test-without-auth.R @@ -109,7 +109,7 @@ test_that("we can get all download urls", { x <- pb_download_url( repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::get_token() + .token = piggyback:::gh::gh_token()() ) expect_is(x, "character") expect_gt(length(x), 1) From 3d190e4d42b3548ff08ee166e9cf21101ebe6925 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 00:02:29 -0400 Subject: [PATCH 10/29] update api-error-msg --- R/gh.R | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/R/gh.R b/R/gh.R index ee3642d..e16ec4c 100644 --- a/R/gh.R +++ b/R/gh.R @@ -14,13 +14,10 @@ local_filename <- function(x) { ##################### Generic helpers ################## api_error_msg <- function(r) { - paste0( - "Cannot access release data for repository ", - crayon::blue$bold(paste0(r[[1]], "/", r[[2]])), - ".", - " Check that you have provided a .token and", - " that at least one release on your GitHub repository page." - ) + cli::cli_warn( + c("!"="Cannot access release data for repo {.val {paste0(r[[1]], "/", r[[2]])}}.", + "Check that you have provided a {code .token} and that there is at least one release on your repo" + )) } ##################################################### From 71c7991edb6eeac3c8d10a6fc27bef9d6b1e22cf Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 00:34:55 -0400 Subject: [PATCH 11/29] some fix crude test rip and replaces, whoops --- tests/testthat/test-errors.R | 2 +- tests/testthat/test-timestamps.R | 4 ++-- tests/testthat/test-with-auth.R | 12 ++++++------ tests/testthat/test-without-auth.R | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/testthat/test-errors.R b/tests/testthat/test-errors.R index 303f3da..48e9936 100644 --- a/tests/testthat/test-errors.R +++ b/tests/testthat/test-errors.R @@ -188,7 +188,7 @@ test_that("download url error", { x <- pb_download_url("not-a-file", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::gh::gh_token()() + .token = gh::gh_token()() ), "not-a-file") }) diff --git a/tests/testthat/test-timestamps.R b/tests/testthat/test-timestamps.R index de74a21..ce1a869 100644 --- a/tests/testthat/test-timestamps.R +++ b/tests/testthat/test-timestamps.R @@ -5,8 +5,8 @@ testthat::test_that( "upload obeys timestamp only when newer", { testthat::skip_on_travis() ## No idea. - testthat::skip_if(piggyback:::gh::gh_token()() == "") - testthat::skip_if(piggyback:::gh::gh_token()() == + testthat::skip_if(gh::gh_token()() == "") + testthat::skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") testthat::skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-with-auth.R b/tests/testthat/test-with-auth.R index 247c1a9..eca6f5a 100644 --- a/tests/testthat/test-with-auth.R +++ b/tests/testthat/test-with-auth.R @@ -6,7 +6,7 @@ context("Requiring Authentication") test_that("We can upload data", { # public pat - skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") data <- readr::write_tsv(datasets::iris, "iris.tsv.gz") @@ -26,7 +26,7 @@ test_that("We can upload data", { test_that("working from git repo", { # public pat - skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") ## Setup @@ -85,13 +85,13 @@ test_that("working from git repo", { test_that("we can get a download url", { # public pat - skip_if(piggyback:::gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") x <- pb_download_url("iris.tsv.gz", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::gh::gh_token()() + .token = gh::gh_token()() ) expect_is(x, "character") }) @@ -114,8 +114,8 @@ test_that( testthat::test_that( "test delete", { - testthat::skip_if(piggyback:::gh::gh_token()() == "") - testthat::skip_if(piggyback:::gh::gh_token()() == + testthat::skip_if(gh::gh_token()() == "") + testthat::skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-without-auth.R b/tests/testthat/test-without-auth.R index 6f0324f..eb797ba 100644 --- a/tests/testthat/test-without-auth.R +++ b/tests/testthat/test-without-auth.R @@ -109,7 +109,7 @@ test_that("we can get all download urls", { x <- pb_download_url( repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = piggyback:::gh::gh_token()() + .token = gh::gh_token()() ) expect_is(x, "character") expect_gt(length(x), 1) From 6e88cfa982552fd1bdd78b37b73fa932cac2133b Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:35:55 -0400 Subject: [PATCH 12/29] memoise pb_releases --- R/zzz.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/R/zzz.R b/R/zzz.R index 4acb9d1..ade047a 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,6 +1,9 @@ .onLoad <- function(libname,pkgname){ - cache_duration <- as.numeric(Sys.getenv("piggyback_cache_duration", "600")) + cache_duration <- as.numeric(Sys.getenv("piggyback_cache_duration",unset = "600")) assign(x = "pb_info", value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), envir = parent.env(environment())) + assign(x = "pb_releases", + value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), + envir = parent.env(environment())) } From e1464c81a2235ce3b81cef9de8352ef00d2397dd Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:37:38 -0400 Subject: [PATCH 13/29] edits to pb_new_release: use pb_releases instead of pb_info --- R/pb_new_release.R | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/R/pb_new_release.R b/R/pb_new_release.R index 2483db0..a0e0dfa 100644 --- a/R/pb_new_release.R +++ b/R/pb_new_release.R @@ -29,9 +29,9 @@ pb_new_release <- function(repo = guess_repo(), body = "Data release", draft = FALSE, prerelease = FALSE, - .token = gh::gh_token()()) { + .token = gh::gh_token()) { - releases <- pb_releases(repo, .token) + releases <- pb_releases(repo = repo, .token = .token, verbose = FALSE) # if no releases exist, pb_releases returns a dataframe of releases if(nrow(releases) > 0 && tag %in% releases$tag_name){ @@ -53,28 +53,25 @@ pb_new_release <- function(repo = guess_repo(), # gh("/repos/:owner/:repo/releases", owner = r[[1]], repo = r[[2]], # .method = "POST", body = toJSON(payload,auto_unbox = TRUE), encode="json") - resp <- httr::POST(paste0( - "https://api.github.com/repos/", r[[1]], "/", - r[[2]], "/", "releases"), + resp <- httr::POST( + glue::glue("https://api.github.com/repos/{r[[1]]}/{r[[2]]}/releases"), httr::add_headers(Authorization = paste("token",.token)), body = jsonlite::toJSON(payload, auto_unbox = TRUE) ) if(httr::http_error(resp)) { - warning( - paste("Failed to create release: HTTP error", - httr::status_code(resp), - "\nSee returned error messages for more details."), - call. = FALSE) - return(invisible(httr::content(resp))) - } + cli::cli_warn( + c("!"="Failed to create release: HTTP error {.val {httr::status_code(resp)}}.", + "See returned error messages for more details")) - ## Release info changed, so break cache - memoise::forget(memoised_pb_info) + return(httr::content(resp)) + } - ## refresh - pb_info(repo = repo, tag = tag, .token = .token) + ## Release info changed, so break caches + memoise::forget(pb_info) + memoise::forget(pb_releases) release <- httr::content(resp) + cli::cli_alert_success() invisible(release) } From 9746b987596e1b315cfa34e8523a4b5c14b2b210 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:40:41 -0400 Subject: [PATCH 14/29] fix pbrelease memoise --- R/pb_new_release.R | 2 +- R/zzz.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/pb_new_release.R b/R/pb_new_release.R index a0e0dfa..1aede56 100644 --- a/R/pb_new_release.R +++ b/R/pb_new_release.R @@ -49,7 +49,7 @@ pb_new_release <- function(repo = guess_repo(), prerelease = prerelease )) - # gh fails to pass body correctly?? + ## gh fails to pass body correctly? # gh("/repos/:owner/:repo/releases", owner = r[[1]], repo = r[[2]], # .method = "POST", body = toJSON(payload,auto_unbox = TRUE), encode="json") diff --git a/R/zzz.R b/R/zzz.R index ade047a..1e025a6 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -4,6 +4,6 @@ value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), envir = parent.env(environment())) assign(x = "pb_releases", - value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), + value = memoise::memoise(pb_releases, ~ memoise::timeout(cache_duration)), envir = parent.env(environment())) } From a3c78f66b7722a5c686ddca2c6b26b71ba8cc790 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:41:12 -0400 Subject: [PATCH 15/29] add verbose flag to pb_releases --- R/pb_info.R | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/R/pb_info.R b/R/pb_info.R index c9bdfd6..933dc60 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -13,7 +13,7 @@ #' @return a dataframe of all releases available within a repository. #' #' @export -pb_releases <- function(repo = guess_repo(), .token = gh::gh_token()){ +pb_releases <- function(repo = guess_repo(), .token = gh::gh_token(), verbose = TRUE){ r <- parse_repo(repo) @@ -28,10 +28,11 @@ pb_releases <- function(repo = guess_repo(), .token = gh::gh_token()){ ) if(length(releases) == 0) { - cli::cli_warn( - c("!" = "No GitHub releases found for {.val {repo}}!", - "You can make a new one with {.fun piggyback::pb_new_release}") - ) + if(verbose){ + cli::cli_warn( + c("!" = "No GitHub releases found for {.val {repo}}!", + "You can make a new one with {.fun piggyback::pb_new_release}") + )} return(data.frame()) } @@ -55,13 +56,13 @@ pb_releases <- function(repo = guess_repo(), .token = gh::gh_token()){ #' @param releases as created by `pb_releases()` #' @param r a list of owner/repo as created by `parse_repo()` #' @noRd -get_release_assets <- function(releases, r) { +get_release_assets <- function(releases, r, .token) { if(nrow(releases)==0) return(data.frame()) asset_list <- vector("list", length = nrow(releases)) - # fetch asset meta-data individually, see #19 + # fetch asset meta-data individually for each release, see #19 for (i in seq_along(releases$tag_name)) { a <- gh::gh(endpoint = "/repos/:owner/:repo/releases/:release_id/assets", owner = r[[1]], @@ -71,9 +72,7 @@ get_release_assets <- function(releases, r) { .token = .token) if(length(a) == 0) next if (!identical(a[[1]], "")) { - # convert list to dataframe and store in asset list - a_df <- data.frame( file_name = vapply(a, `[[`, character(1), "name"), size = vapply(a, `[[`, integer(1), "size"), @@ -106,7 +105,7 @@ pb_info <- function(repo = guess_repo(), r <- parse_repo(repo) # get all releases - releases <- pb_releases(repo, .token) + releases <- pb_releases(repo = repo, .token = .token, verbose = FALSE) # if no releases return empty df if(nrow(releases) == 0) { @@ -126,14 +125,14 @@ pb_info <- function(repo = guess_repo(), )) } - # if tag is latest, do call for first tag only + # if tag is latest, set tag to first tag present in releases if(!is.null(tag) && length(tag) == 1 && tag == "latest") tag <- releases$tag_name[[1]] # if tag is present, filter the releases to search to just the tags requested if(!is.null(tag)) releases <- releases[releases$tag_name %in% tag,] # get release assets and metadata for each release - info <- get_release_assets(releases, r) + info <- get_release_assets(releases = releases, r = r, .token = .token) return(info) } From 323d5ce39021ac3e08c627494010f2eebd16b290 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:42:19 -0400 Subject: [PATCH 16/29] fix stupid replacement --- R/pb_download.R | 4 ++-- R/pb_download_url.R | 2 +- R/pb_list.R | 2 +- R/pb_upload.R | 4 ++-- man/pb_download.Rd | 2 +- man/pb_download_url.Rd | 2 +- man/pb_list.Rd | 5 ++++- man/pb_new_release.Rd | 2 +- man/pb_releases.Rd | 2 +- man/pb_upload.Rd | 2 +- 10 files changed, 15 insertions(+), 12 deletions(-) diff --git a/R/pb_download.R b/R/pb_download.R index eeb30f2..bd775f6 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -35,7 +35,7 @@ pb_download <- function(file = NULL, ignore = "manifest.json", use_timestamps = TRUE, show_progress = TRUE, - .token = gh::gh_token()()) { + .token = gh::gh_token()) { progress <- httr::progress("down") if (!show_progress) { progress <- NULL @@ -111,7 +111,7 @@ gh_download_asset <- function(owner, id, destfile, overwrite = TRUE, - .token = gh::gh_token()(), + .token = gh::gh_token(), progress = httr::progress("down")) { if (fs::file_exists(destfile) && !overwrite) { warning(paste( diff --git a/R/pb_download_url.R b/R/pb_download_url.R index 4991a0c..b287051 100644 --- a/R/pb_download_url.R +++ b/R/pb_download_url.R @@ -17,7 +17,7 @@ pb_download_url <- function(file = NULL, repo = guess_repo(), tag = "latest", - .token = gh::gh_token()()) { + .token = gh::gh_token()) { df <- pb_info(repo, tag, .token) if (is.null(file)) { return(df$browser_download_url) diff --git a/R/pb_list.R b/R/pb_list.R index 2470d6e..8389e09 100644 --- a/R/pb_list.R +++ b/R/pb_list.R @@ -11,7 +11,7 @@ #' @export pb_list <- function(repo = guess_repo(), tag = NULL, - .token = gh::gh_token()()) { + .token = gh::gh_token()) { df <- pb_info(repo, tag, .token) df[c("file_name", "size", "timestamp", "tag", "owner", "repo")] } diff --git a/R/pb_upload.R b/R/pb_upload.R index 41b07e5..2ac6b29 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -58,7 +58,7 @@ pb_upload <- function(file, if(run == "No") return(invisible(NULL)) - if(run == "Yes") pb_new_release(repo = repo, tag = tag, .token = token) + if(run == "Yes") pb_new_release(repo = repo, tag = tag, .token = .token) } ## start fresh @@ -89,7 +89,7 @@ pb_upload_file <- function(file, overwrite = "use_timestamps", use_timestamps = NULL, show_progress = TRUE, - .token = gh::gh_token()(), + .token = gh::gh_token(), dir = NULL) { file_path <- do.call(file.path, compact(list(dir,file))) diff --git a/man/pb_download.Rd b/man/pb_download.Rd index 451c9a4..dbe151e 100644 --- a/man/pb_download.Rd +++ b/man/pb_download.Rd @@ -13,7 +13,7 @@ pb_download( ignore = "manifest.json", use_timestamps = TRUE, show_progress = TRUE, - .token = get_token() + .token = gh::gh_token() ) } \arguments{ diff --git a/man/pb_download_url.Rd b/man/pb_download_url.Rd index 18f6041..5885a3a 100644 --- a/man/pb_download_url.Rd +++ b/man/pb_download_url.Rd @@ -8,7 +8,7 @@ pb_download_url( file = NULL, repo = guess_repo(), tag = "latest", - .token = get_token() + .token = gh::gh_token() ) } \arguments{ diff --git a/man/pb_list.Rd b/man/pb_list.Rd index 04eaa5d..7614952 100644 --- a/man/pb_list.Rd +++ b/man/pb_list.Rd @@ -4,7 +4,7 @@ \alias{pb_list} \title{List all assets attached to a release} \usage{ -pb_list(repo = guess_repo(), tag = NULL, .token = get_token()) +pb_list(repo = guess_repo(), tag = NULL, .token = gh::gh_token()) } \arguments{ \item{repo}{Repository name in format "owner/repo". Will guess the current @@ -26,3 +26,6 @@ List all assets attached to a release pb_list("cboettig/piggyback-tests") } } +\seealso{ +\code{pb_releases} for a list of all releases in repository +} diff --git a/man/pb_new_release.Rd b/man/pb_new_release.Rd index 4b93ad0..c801fa9 100644 --- a/man/pb_new_release.Rd +++ b/man/pb_new_release.Rd @@ -12,7 +12,7 @@ pb_new_release( body = "Data release", draft = FALSE, prerelease = FALSE, - .token = get_token() + .token = gh::gh_token() ) } \arguments{ diff --git a/man/pb_releases.Rd b/man/pb_releases.Rd index c1b5213..e657b56 100644 --- a/man/pb_releases.Rd +++ b/man/pb_releases.Rd @@ -4,7 +4,7 @@ \alias{pb_releases} \title{List releases in repository} \usage{ -pb_releases(repo = guess_repo(), .token = gh::gh_token()) +pb_releases(repo = guess_repo(), .token = gh::gh_token(), verbose = TRUE) } \arguments{ \item{repo}{GitHub repository specification in the form of \code{"owner/repo"}, if not specified will try to guess repo based on current working directory.} diff --git a/man/pb_upload.Rd b/man/pb_upload.Rd index 387a672..8ccf7ed 100644 --- a/man/pb_upload.Rd +++ b/man/pb_upload.Rd @@ -12,7 +12,7 @@ pb_upload( overwrite = "use_timestamps", use_timestamps = NULL, show_progress = TRUE, - .token = get_token(), + .token = gh::gh_token(), dir = NULL ) } From ef730e464675a3a8a8fb40b635970940b53181d9 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 08:55:05 -0400 Subject: [PATCH 17/29] cleanup unused utils --- NAMESPACE | 2 - R/gh.R | 33 +++++------ R/pb_info.R | 7 ++- R/utils.R | 162 ++++++++++++++++++++++++++-------------------------- 4 files changed, 101 insertions(+), 103 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 941e476..e9fd0f0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -17,8 +17,6 @@ importFrom(httr,GET) importFrom(httr,POST) importFrom(httr,add_headers) importFrom(httr,content) -importFrom(httr,progress) importFrom(httr,stop_for_status) -importFrom(httr,upload_file) importFrom(httr,write_disk) importFrom(jsonlite,toJSON) diff --git a/R/gh.R b/R/gh.R index e16ec4c..d8d24a4 100644 --- a/R/gh.R +++ b/R/gh.R @@ -1,26 +1,23 @@ -asset_filename <- function(x, start = ".") { - ## piggyback will no longer embed file structure in filename - ## Asset uploading is simply flat storage - x -} - -local_filename <- function(x) { - # x <- gsub("^manifest.json$", ".manifest.json", x) +# asset_filename <- function(x, start = ".") { +# ## piggyback will no longer embed file structure in filename +# ## Asset uploading is simply flat storage +# x +# } - gsub("\\.2f", .Platform$file.sep, x) -} +# local_filename <- function(x) { +# # x <- gsub("^manifest.json$", ".manifest.json", x) +# gsub("\\.2f", .Platform$file.sep, x) +# } ##################### Generic helpers ################## -api_error_msg <- function(r) { - cli::cli_warn( - c("!"="Cannot access release data for repo {.val {paste0(r[[1]], "/", r[[2]])}}.", - "Check that you have provided a {code .token} and that there is at least one release on your repo" - )) -} - -##################################################### +# api_error_msg <- function(r) { +# cli::cli_warn( +# c("!"="Cannot access release data for repo {.val {paste0(r[[1]], '/', r[[2]])}}.", +# "Check that you have provided a {code .token} and that there is at least one release on your repo" +# )) +# } maybe <- function(expr, otherwise, quiet = TRUE) { if (missing(otherwise)) { diff --git a/R/pb_info.R b/R/pb_info.R index 933dc60..932735d 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -24,7 +24,10 @@ pb_releases <- function(repo = guess_repo(), .token = gh::gh_token(), verbose = repo = r[[2]], .limit = Inf, .token = .token), - otherwise = stop(api_error_msg(r)) + otherwise = cli::cli_abort( + c("!"="Cannot access release data for repo {.val {repo}}.", + "Check that you have provided a {.code .token} and that the repo is correctly specified.") + ) ) if(length(releases) == 0) { @@ -81,7 +84,7 @@ get_release_assets <- function(releases, r, .token) { owner = r[[1]], repo = r[[2]], upload_url = releases$upload_url[i], - browser_download_url = vapply(a, `[[`, character(1L),"browser_download_url"), + browser_download_url = vapply(a, `[[`, character(1L), "browser_download_url"), id = vapply(a, `[[`, integer(1L), "id"), state = vapply(a, `[[`, character(1L), "state"), stringsAsFactors = FALSE diff --git a/R/utils.R b/R/utils.R index 37ef0fd..a9c030e 100644 --- a/R/utils.R +++ b/R/utils.R @@ -13,91 +13,91 @@ parse_repo <- function(repo){ return(r) } +#' Remove null elements of a list +#' @keywords internal +#' @noRd +compact <- function (l) Filter(Negate(is.null), l) # Adapted from `usethis` and under # GPL-3 (Copyright RStudio Inc) # https://github.com/r-lib/usethis/blob/aacf687445c94d4f726e31f682308a9a3ef3f820/R/write.R # See https://github.com/r-lib/usethis/issues/366 -write_union <- function(base_path, path, new_lines, quiet = FALSE) { - stopifnot(is.character(new_lines)) - - full_path <- file.path(base_path, path) - if (file.exists(full_path)) { - lines <- readLines(full_path, warn = FALSE) - } else { - lines <- character() - } - - new <- setdiff(new_lines, lines) - if (length(new) == 0) { - return(invisible(FALSE)) - } - - if (!quiet) { - quoted <- paste0(value(new), collapse = ", ") - done("Adding ", quoted, " to ", value(path)) - } - - all <- union(lines, new_lines) - write_utf8(full_path, all) -} - -write_utf8 <- function(path, lines) { - stopifnot(is.character(path)) - stopifnot(is.character(lines)) - - con <- file(path, encoding = "utf-8") - on.exit(close(con), add = TRUE) - - if (length(lines) > 1) { - lines <- paste0(lines, "\n", collapse = "") - } - cat(lines, file = con, sep = "") - - invisible(TRUE) -} - -#' @importFrom crayon green blue -#' @importFrom clisymbols symbol -done <- function(...) { - bullet(paste0(...), bullet = crayon::green(clisymbols::symbol$tick)) -} - -value <- function(...) { - x <- paste0(...) - crayon::blue(encodeString(x, quote = "'")) -} - -bullet <- function(lines, bullet) { - lines <- paste0(bullet, " ", lines) - cat_line(lines) -} - -cat_line <- function(...) { - cat(..., "\n", sep = "") -} - -compact <- function (l) Filter(Negate(is.null), l) +# write_union <- function(base_path, path, new_lines, quiet = FALSE) { +# stopifnot(is.character(new_lines)) +# +# full_path <- file.path(base_path, path) +# if (file.exists(full_path)) { +# lines <- readLines(full_path, warn = FALSE) +# } else { +# lines <- character() +# } +# +# new <- setdiff(new_lines, lines) +# if (length(new) == 0) { +# return(invisible(FALSE)) +# } +# +# if (!quiet) { +# quoted <- paste0(value(new), collapse = ", ") +# done("Adding ", quoted, " to ", value(path)) +# } +# +# all <- union(lines, new_lines) +# write_utf8(full_path, all) +# } +# +# write_utf8 <- function(path, lines) { +# stopifnot(is.character(path)) +# stopifnot(is.character(lines)) +# +# con <- file(path, encoding = "utf-8") +# on.exit(close(con), add = TRUE) +# +# if (length(lines) > 1) { +# lines <- paste0(lines, "\n", collapse = "") +# } +# cat(lines, file = con, sep = "") +# +# invisible(TRUE) +# } +# +# done <- function(...) { +# bullet(paste0(...), bullet = crayon::green(clisymbols::symbol$tick)) +# } +# +# value <- function(...) { +# x <- paste0(...) +# crayon::blue(encodeString(x, quote = "'")) +# } +# +# bullet <- function(lines, bullet) { +# lines <- paste0(bullet, " ", lines) +# cat_line(lines) +# } +# +# cat_line <- function(...) { +# cat(..., "\n", sep = "") +# } # utils::askYesKnow is new to R 3.5.0; avoid using it for backwards compatibility -askYesNo <- function(msg){ - - prompts <- c("Yes", "No", "Cancel") - choices <- tolower(prompts) - msg1 <- paste0("(", paste(choices, collapse = "/"), ") ") - - if (nchar(paste0(msg, msg1)) > 250) { - cat(msg, "\n") - msg <- msg1 - } - else msg <- paste0(msg, " ", msg1) - - ans <- readline(msg) - match <- pmatch(tolower(ans), tolower(choices)) - - if (!nchar(ans)) - TRUE - else if (is.na(match)) - stop("Unrecognized response ", dQuote(ans)) - else c(TRUE, FALSE, NA)[match] -} +# askYesNo <- function(msg){ +# +# prompts <- c("Yes", "No", "Cancel") +# choices <- tolower(prompts) +# msg1 <- paste0("(", paste(choices, collapse = "/"), ") ") +# +# if (nchar(paste0(msg, msg1)) > 250) { +# cat(msg, "\n") +# msg <- msg1 +# } +# else msg <- paste0(msg, " ", msg1) +# +# ans <- readline(msg) +# match <- pmatch(tolower(ans), tolower(choices)) +# +# if (!nchar(ans)) +# TRUE +# else if (is.na(match)) +# stop("Unrecognized response ", dQuote(ans)) +# else c(TRUE, FALSE, NA)[match] +# } From d8dd1aec70b7702b7f8f0311b537052d6e3d3cf7 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 09:07:02 -0400 Subject: [PATCH 18/29] start fixing tests --- tests/testthat/test-errors.R | 15 ++++----------- tests/testthat/test-timestamps.R | 4 ++-- tests/testthat/test-with-auth.R | 12 ++++++------ tests/testthat/test-without-auth.R | 2 +- 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/tests/testthat/test-errors.R b/tests/testthat/test-errors.R index 48e9936..a890a2b 100644 --- a/tests/testthat/test-errors.R +++ b/tests/testthat/test-errors.R @@ -23,9 +23,6 @@ test_that( } ) - - - test_that( "Attempt overwrite on upload when overwrite is FALSE", { skip_on_cran() @@ -39,7 +36,7 @@ test_that( overwrite = FALSE, show_progress = FALSE ), - "Skipping upload of iris.tsv.gz as file exists" + "Skipping upload .+ as file exists" ) unlink("iris.tsv.gz") } @@ -59,7 +56,7 @@ test_that( use_timestamps = FALSE, show_progress = FALSE ), - "not-a-file does not exist" + "file .+ does not exist" ) } @@ -91,8 +88,6 @@ test_that( ## Note: in interactive use this will prompt instead skip_if(interactive()) - - path <- file.path(tmp, "iris.tsv.gz") readr::write_tsv(datasets::iris, path) @@ -104,7 +99,7 @@ test_that( overwrite = TRUE, show_progress = FALSE ), - "No release with tag not-a-tag exists" + "Release .* not found" ) @@ -182,13 +177,11 @@ test_that( test_that("download url error", { skip_on_cran() - - expect_error( x <- pb_download_url("not-a-file", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = gh::gh_token()() + .token = gh::gh_token() ), "not-a-file") }) diff --git a/tests/testthat/test-timestamps.R b/tests/testthat/test-timestamps.R index ce1a869..b9bc3b2 100644 --- a/tests/testthat/test-timestamps.R +++ b/tests/testthat/test-timestamps.R @@ -5,8 +5,8 @@ testthat::test_that( "upload obeys timestamp only when newer", { testthat::skip_on_travis() ## No idea. - testthat::skip_if(gh::gh_token()() == "") - testthat::skip_if(gh::gh_token()() == + testthat::skip_if(gh::gh_token() == "") + testthat::skip_if(gh::gh_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") testthat::skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-with-auth.R b/tests/testthat/test-with-auth.R index eca6f5a..76cadaf 100644 --- a/tests/testthat/test-with-auth.R +++ b/tests/testthat/test-with-auth.R @@ -6,7 +6,7 @@ context("Requiring Authentication") test_that("We can upload data", { # public pat - skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") data <- readr::write_tsv(datasets::iris, "iris.tsv.gz") @@ -26,7 +26,7 @@ test_that("We can upload data", { test_that("working from git repo", { # public pat - skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") ## Setup @@ -85,13 +85,13 @@ test_that("working from git repo", { test_that("we can get a download url", { # public pat - skip_if(gh::gh_token()() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") + skip_if(gh::gh_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") x <- pb_download_url("iris.tsv.gz", repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = gh::gh_token()() + .token = gh::gh_token() ) expect_is(x, "character") }) @@ -114,8 +114,8 @@ test_that( testthat::test_that( "test delete", { - testthat::skip_if(gh::gh_token()() == "") - testthat::skip_if(gh::gh_token()() == + testthat::skip_if(gh::gh_token() == "") + testthat::skip_if(gh::gh_token() == "b2b7441daeeb010b1df26f1f60a7f1edc485e443") skip_if(Sys.getenv("CBOETTIG_TOKEN") == "") diff --git a/tests/testthat/test-without-auth.R b/tests/testthat/test-without-auth.R index eb797ba..036f2cb 100644 --- a/tests/testthat/test-without-auth.R +++ b/tests/testthat/test-without-auth.R @@ -109,7 +109,7 @@ test_that("we can get all download urls", { x <- pb_download_url( repo = "cboettig/piggyback-tests", tag = "v0.0.1", - .token = gh::gh_token()() + .token = gh::gh_token() ) expect_is(x, "character") expect_gt(length(x), 1) From fdf58c0e8d3e2d87d0c4a1f50f7264cebed57be1 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 09:36:30 -0400 Subject: [PATCH 19/29] cleanup crayon stuff --- DESCRIPTION | 4 ++-- NAMESPACE | 3 --- R/pb_delete.R | 2 +- R/pb_download.R | 31 ++++++++++++++----------------- R/pb_info.R | 2 ++ man/pb_delete.Rd | 2 +- man/pb_releases.Rd | 4 +++- 7 files changed, 23 insertions(+), 25 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 571991d..d52b409 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -33,12 +33,12 @@ License: GPL-3 Encoding: UTF-8 ByteCompile: true Imports: + cli, + glue, gh, httr, jsonlite, fs, - crayon, - clisymbols, lubridate, memoise Suggests: diff --git a/NAMESPACE b/NAMESPACE index e9fd0f0..b6b11f4 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,9 +7,6 @@ export(pb_list) export(pb_new_release) export(pb_releases) export(pb_upload) -importFrom(clisymbols,symbol) -importFrom(crayon,blue) -importFrom(crayon,green) importFrom(fs,dir_create) importFrom(gh,gh) importFrom(gh,gh_token) diff --git a/R/pb_delete.R b/R/pb_delete.R index 240f34b..8494352 100644 --- a/R/pb_delete.R +++ b/R/pb_delete.R @@ -13,7 +13,7 @@ #' ## Upload #' pb_upload("mtcars.tsv.gz", #' repo = "cboettig/piggyback-tests", -#' overwrite = TRUE) +#' overwrite = TRUE) #' pb_delete("mtcars.tsv.gz", #' repo = "cboettig/piggyback-tests", #' tag = "v0.0.1") diff --git a/R/pb_download.R b/R/pb_download.R index bd775f6..0bece4a 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -50,12 +50,10 @@ pb_download <- function(file = NULL, if (!is.null(file)) { i <- which(df$file_name %in% file) if (length(i) < 1) { - warning(paste( - "file(s)", - paste(crayon::blue(file), collapse = " "), - "not found in repo", - crayon::blue(repo) - )) + + cli::cli_warn( + "file(s) {.file {file}} not found in repo {.val {repo}}" + ) } df <- df[i, ] @@ -86,7 +84,7 @@ pb_download <- function(file = NULL, df <- df[update, ] if (dim(df)[[1]] < 1) { - message(paste("All files up-to-date already\n")) + cli::cli_alert_info("All local files already up-to-date!") } } @@ -101,11 +99,8 @@ pb_download <- function(file = NULL, invisible(resp) } - - - ## gh() fails on this, so we do with httr. See https://github.com/r-lib/gh/issues/57 -## Consider option to supress progress bar? +## Consider option to suppress progress bar? gh_download_asset <- function(owner, repo, id, @@ -114,15 +109,17 @@ gh_download_asset <- function(owner, .token = gh::gh_token(), progress = httr::progress("down")) { if (fs::file_exists(destfile) && !overwrite) { - warning(paste( - destfile, "already exists, skipping download.", - "Set overwrite = TRUE to overwrite files." - )) + cli::cli_warn( + c("!"="{.val {destfile}} already exists, skipping download.", + "Set {.code overwrite = TRUE} to overwrite files.") + ) return(NULL) } if (!is.null(progress)) { - message(paste("downloading", basename(destfile), "...")) + cli::cli_alert_info( + "Downloading {.val {basename(destfile)}}..." + ) } resp <- httr::GET( @@ -149,7 +146,7 @@ gh_download_asset <- function(owner, } # handle error cases? resp not found - if(getOption("verbose")) httr::warn_for_status(resp) + if(getOption("piggyback.verbose")) httr::warn_for_status(resp) invisible(resp) # gh::gh(paste0( diff --git a/R/pb_info.R b/R/pb_info.R index 932735d..fe5cbe9 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -1,8 +1,10 @@ #' List releases in repository #' +#' This function retrieves information about all releases attached to a given repository. #' #' @param repo GitHub repository specification in the form of `"owner/repo"`, if not specified will try to guess repo based on current working directory. #' @param .token a GitHub API token, defaults to `gh::gh_token()` +#' @param verbose defaults to TRUE, use FALSE to silence messages #' #' @examples #' \donttest{ diff --git a/man/pb_delete.Rd b/man/pb_delete.Rd index 99d0955..a8eefc3 100644 --- a/man/pb_delete.Rd +++ b/man/pb_delete.Rd @@ -36,7 +36,7 @@ readr::write_tsv(mtcars, "mtcars.tsv.gz") ## Upload pb_upload("mtcars.tsv.gz", repo = "cboettig/piggyback-tests", - overwrite = TRUE) + overwrite = TRUE) pb_delete("mtcars.tsv.gz", repo = "cboettig/piggyback-tests", tag = "v0.0.1") diff --git a/man/pb_releases.Rd b/man/pb_releases.Rd index e657b56..6e5de01 100644 --- a/man/pb_releases.Rd +++ b/man/pb_releases.Rd @@ -10,12 +10,14 @@ pb_releases(repo = guess_repo(), .token = gh::gh_token(), verbose = TRUE) \item{repo}{GitHub repository specification in the form of \code{"owner/repo"}, if not specified will try to guess repo based on current working directory.} \item{.token}{a GitHub API token, defaults to \code{gh::gh_token()}} + +\item{verbose}{defaults to TRUE, use FALSE to silence messages} } \value{ a dataframe of all releases available within a repository. } \description{ -List releases in repository +This function retrieves information about all releases attached to a given repository. } \examples{ \donttest{ From c3775eadca9dbc21a7d393e066b24808d4e9654e Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 19:10:11 -0400 Subject: [PATCH 20/29] pb upload messaging --- R/pb_upload.R | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/R/pb_upload.R b/R/pb_upload.R index 2ac6b29..053f132 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -43,14 +43,13 @@ pb_upload <- function(file, releases <- pb_releases(repo, .token) + if(tag != "latest" && !tag %in% releases$tag_name && !interactive()) { + cli::cli_abort("Release {.val {tag}} not found in {.val {repo}}. No upload performed.") + } + if(tag != "latest" && !tag %in% releases$tag_name) { cli::cli_alert_warning("Release {.val {tag}} not found in {.val {repo}}.") - if(!interactive()) { - cli::cli_alert_warning("No upload performed.") - return(invisible(NULL)) - } - run <- utils::menu( choices = c("Yes", "No"), title = glue::glue("Would you like to create a new release now?") From 3e23e9a28f3d60890a11f2fccd5b2e2fc43a17ab Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 19:17:11 -0400 Subject: [PATCH 21/29] pb_download_url better handles multiple files, including missings --- R/pb_download_url.R | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/R/pb_download_url.R b/R/pb_download_url.R index b287051..ce4ad85 100644 --- a/R/pb_download_url.R +++ b/R/pb_download_url.R @@ -19,11 +19,19 @@ pb_download_url <- function(file = NULL, tag = "latest", .token = gh::gh_token()) { df <- pb_info(repo, tag, .token) - if (is.null(file)) { - return(df$browser_download_url) - } else if (file %in% df$file_name) { - return(df[file == df$file_name, "browser_download_url"]) - } else { - stop(paste("file", file, "not found in release", tag, "for repo", repo)) + + if(is.null(file)) return(df$browser_download_url) + + if(any(!file %in% df$file_name)) { + + missing <- file[!file %in% df$file_name] + + cli::cli_warn("file {.val {missing}} not found in release {.val {tag}} for repo {.val {repo}}") + + file <- file[file %in% df$file_name] } + + if(length(file) == 0) return(cli::cli_abort("No download URLs to return.")) + + return(df[df$file_name %in% file,"browser_download_url"]) } From 8e7374aa82f8835b113787c5c0d8153a9063e608 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 19:24:08 -0400 Subject: [PATCH 22/29] fixup verbose option usage --- R/pb_download.R | 6 +++--- R/pb_download_url.R | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/R/pb_download.R b/R/pb_download.R index 0bece4a..7a49cfb 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -67,11 +67,10 @@ pb_download <- function(file = NULL, ## if dest paths are not provided, we will write all files to dest dir + # User is responsible for making sure dest dir exists! if (length(dest) == 1) { i <- which(df$file_name %in% file) dest <- file.path(dest, df$file_name[i]) - # User is responsible for making sure dest dir exists! - # fs::dir_create(fs::path_dir(dest)) } # dest should now be of length df df$dest <- dest @@ -146,9 +145,10 @@ gh_download_asset <- function(owner, } # handle error cases? resp not found - if(getOption("piggyback.verbose")) httr::warn_for_status(resp) + if(getOption("piggyback.verbose", default = TRUE)) httr::warn_for_status(resp) invisible(resp) + # gh::gh(paste0( # "https://api.github.com/repos/", owner, "/", # repo, "/", "releases/assets/", id), diff --git a/R/pb_download_url.R b/R/pb_download_url.R index ce4ad85..15c4947 100644 --- a/R/pb_download_url.R +++ b/R/pb_download_url.R @@ -1,4 +1,3 @@ - #' Get the download url of a given file #' #' Returns the URL download for a public file. This can be useful when writing From ec3264f1bdf922f06bd72383f962a677416320dd Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 19:31:35 -0400 Subject: [PATCH 23/29] edits to tests to reflect new error messaging --- tests/testthat/test-errors.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/testthat/test-errors.R b/tests/testthat/test-errors.R index a890a2b..43fa30e 100644 --- a/tests/testthat/test-errors.R +++ b/tests/testthat/test-errors.R @@ -145,13 +145,13 @@ test_that( skip_on_cran() - expect_message( + expect_warning( pb_delete( repo = "cboettig/piggyback-tests", file = "mtcars8.tsv.gz", tag = "v0.0.1" ), - "not found on GitHub" + "No file deletions performed" ) } ) @@ -177,11 +177,11 @@ test_that( test_that("download url error", { skip_on_cran() - expect_error( - x <- pb_download_url("not-a-file", - repo = "cboettig/piggyback-tests", - tag = "v0.0.1", - .token = gh::gh_token() - ), "not-a-file") + expect_warning( + expect_error( + pb_download_url("not-a-file", repo = "cboettig/piggyback-tests", tag = "v0.0.1", .token = gh::gh_token()), + "No download URLs to return"), + "file .+ not found in release" + ) }) From 444314da45d9706cb6280e15b6119bd225cf2aa7 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 22:59:21 -0400 Subject: [PATCH 24/29] minor cleanups --- R/pb_download.R | 5 +---- R/pb_upload.R | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/R/pb_download.R b/R/pb_download.R index 7a49cfb..8a7d1a2 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -50,10 +50,7 @@ pb_download <- function(file = NULL, if (!is.null(file)) { i <- which(df$file_name %in% file) if (length(i) < 1) { - - cli::cli_warn( - "file(s) {.file {file}} not found in repo {.val {repo}}" - ) + cli::cli_warn("file(s) {.file {file}} not found in repo {.val {repo}}") } df <- df[i, ] diff --git a/R/pb_upload.R b/R/pb_upload.R index 053f132..8eb1f84 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -56,7 +56,6 @@ pb_upload <- function(file, ) if(run == "No") return(invisible(NULL)) - if(run == "Yes") pb_new_release(repo = repo, tag = tag, .token = .token) } @@ -173,7 +172,7 @@ pb_upload_file <- function(file, cat("\n") - if(getOption("piggyback.verbose")) httr::warn_for_status(r) + if(getOption("piggyback.verbose",default = TRUE)) httr::warn_for_status(r) ## Release info changed, so break cache memoise::forget(pb_info) From 93b4b0b8302135b285592536d1c773bf5ebb38f1 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 23:18:40 -0400 Subject: [PATCH 25/29] incorporate change from #62 --- R/pb_download.R | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/R/pb_download.R b/R/pb_download.R index 8a7d1a2..7900487 100644 --- a/R/pb_download.R +++ b/R/pb_download.R @@ -10,9 +10,7 @@ #' @param ignore a list of files to ignore (if downloading "all" because #' `file=NULL`). #' @inheritParams pb_upload -#' @importFrom httr GET add_headers write_disk -#' @importFrom gh gh gh_token -#' @importFrom fs dir_create +#' #' @export #' @examples \dontrun{ #' ## Download a specific file. @@ -36,11 +34,10 @@ pb_download <- function(file = NULL, use_timestamps = TRUE, show_progress = TRUE, .token = gh::gh_token()) { + progress <- httr::progress("down") - if (!show_progress) { - progress <- NULL - } + if (!show_progress) progress <- NULL df <- pb_info(repo, tag, .token) @@ -88,7 +85,7 @@ pb_download <- function(file = NULL, gh_download_asset(df$owner[[1]], df$repo[[1]], id = df$id[i], - destfile = dest[i], + destfile = df$dest[i], overwrite = overwrite, progress = progress )) From c97f73b22a9be007b00826effd507de455d54b58 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 23:22:07 -0400 Subject: [PATCH 26/29] update news --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index a103f6f..2fa63d5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,12 @@ * update intro vignette to remove mention of path name handling and instead provide examples of how path names are handled. * update intro vignette instructions for git authentication * `pb_new_release()` now reports HTTP errors when attempting to create a new release and returns the contents of the error if it fails. +* `pb_releases()` created - it returns a list of releases available in the repository. +* Internal function `pb_info()` refactored to search for the specified tag(s) which should improve performance. Should handle multiple tags gracefully. +* Internal function `pb_info()` (and therefore `pb_list()`, `pb_download()`, `pb_download_url()`) no longer ask about creating new releases if the release is not found. +* `pb_upload()` is now the only function that offers (interactively) to create a new release if release is not found. If noninteractive, user must run `pb_new_release()` manually prior to uploading. +* CLI messaging now consistently uses `{cli}` package and no longer uses clisymbols or crayon - this is to align with the imports from the `{gh}` package. +* Documentation updated. # piggyback 0.1.1 From 1c14afdb8abe67a3dc546b639c1c430173cf9daa Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 23:22:25 -0400 Subject: [PATCH 27/29] move example files out of inst and into dev --- .Rbuildignore | 1 + {inst => dev}/examples/pb_track.R | 0 {inst => dev}/examples/push-pull.R | 0 {inst => dev}/examples/test-deprecated.R | 0 {inst => dev}/examples/upload-test-files.R | 0 5 files changed, 1 insertion(+) rename {inst => dev}/examples/pb_track.R (100%) rename {inst => dev}/examples/push-pull.R (100%) rename {inst => dev}/examples/test-deprecated.R (100%) rename {inst => dev}/examples/upload-test-files.R (100%) diff --git a/.Rbuildignore b/.Rbuildignore index 10934a9..0599d2a 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -21,6 +21,7 @@ ^paper\.bib$ ^codemeta\.json$ ^docs$ +^dev$ ^.*\.gz$ mtcars.tsv.gz$ ^\.github$ diff --git a/inst/examples/pb_track.R b/dev/examples/pb_track.R similarity index 100% rename from inst/examples/pb_track.R rename to dev/examples/pb_track.R diff --git a/inst/examples/push-pull.R b/dev/examples/push-pull.R similarity index 100% rename from inst/examples/push-pull.R rename to dev/examples/push-pull.R diff --git a/inst/examples/test-deprecated.R b/dev/examples/test-deprecated.R similarity index 100% rename from inst/examples/test-deprecated.R rename to dev/examples/test-deprecated.R diff --git a/inst/examples/upload-test-files.R b/dev/examples/upload-test-files.R similarity index 100% rename from inst/examples/upload-test-files.R rename to dev/examples/upload-test-files.R From fa83be808a37191c1e7d5b4738889c27b3a236da Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 23:34:38 -0400 Subject: [PATCH 28/29] updates to documentation --- NAMESPACE | 10 --- R/gh.R | 68 ------------------ R/pb_info.R | 6 +- R/pb_new_release.R | 10 +-- R/pb_upload.R | 8 +-- R/piggyback.R | 5 -- R/utils.R | 138 +++++++++++++++---------------------- R/zzz.R | 2 +- README.Rmd | 6 +- README.md | 2 +- codemeta.json | 48 +++++++------ dev/examples/gh.R | 105 ++++++++++++++++++++++++++++ man/pb_releases.Rd | 6 +- man/piggyback-package.Rd | 7 -- vignettes/alternatives.Rmd | 4 -- vignettes/intro.Rmd | 2 +- 16 files changed, 208 insertions(+), 219 deletions(-) delete mode 100644 R/gh.R create mode 100644 dev/examples/gh.R diff --git a/NAMESPACE b/NAMESPACE index b6b11f4..e3961f5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,13 +7,3 @@ export(pb_list) export(pb_new_release) export(pb_releases) export(pb_upload) -importFrom(fs,dir_create) -importFrom(gh,gh) -importFrom(gh,gh_token) -importFrom(httr,GET) -importFrom(httr,POST) -importFrom(httr,add_headers) -importFrom(httr,content) -importFrom(httr,stop_for_status) -importFrom(httr,write_disk) -importFrom(jsonlite,toJSON) diff --git a/R/gh.R b/R/gh.R deleted file mode 100644 index d8d24a4..0000000 --- a/R/gh.R +++ /dev/null @@ -1,68 +0,0 @@ - -# asset_filename <- function(x, start = ".") { -# ## piggyback will no longer embed file structure in filename -# ## Asset uploading is simply flat storage -# x -# } - -# local_filename <- function(x) { -# # x <- gsub("^manifest.json$", ".manifest.json", x) -# gsub("\\.2f", .Platform$file.sep, x) -# } - - -##################### Generic helpers ################## -# api_error_msg <- function(r) { -# cli::cli_warn( -# c("!"="Cannot access release data for repo {.val {paste0(r[[1]], '/', r[[2]])}}.", -# "Check that you have provided a {code .token} and that there is at least one release on your repo" -# )) -# } - -maybe <- function(expr, otherwise, quiet = TRUE) { - if (missing(otherwise)) { - try(expr, silent = quiet) - } else { - tryCatch(expr, - error = function(e) { - if (!quiet) { - message("Error: ", e$message) - } - otherwise - } - ) - } -} - - -guess_repo <- function(path = ".") { - - exists <- requireNamespace("gert", quietly = TRUE) - if(!exists) stop(paste( - "Install package 'gert' to let piggyback discover the", - "current repo, or provide your repo name explicitly")) - - repo <- gert::git_find(path) - remotes <- gert::git_remote_list(repo) - remotes_names <- remotes[["name"]] - - # When there are more than 1 remote, we prefer "upstream" - # then "origin." If neither exists, we error to avoid - # ambiguity. - remote <- if (length(remotes_names) > 1) { - if ("upstream" %in% remotes_names) { - "upstream" - } else if ("origin" %in% remotes_names) { - "origin" - } else - stop("Cannot infer repo, please provide `repo` explicitly.", - call. = FALSE) - } else { - remotes_names - } - - addr <- remotes[remotes[["name"]] == remote, "url"][["url"]] - - out <- gsub(".*[:|/]([^/]+/[^/]+)(?:\\.git$)?", "\\1", addr) - gsub("\\.git$", "", out) -} diff --git a/R/pb_info.R b/R/pb_info.R index fe5cbe9..de29417 100644 --- a/R/pb_info.R +++ b/R/pb_info.R @@ -15,7 +15,9 @@ #' @return a dataframe of all releases available within a repository. #' #' @export -pb_releases <- function(repo = guess_repo(), .token = gh::gh_token(), verbose = TRUE){ +pb_releases <- function(repo = guess_repo(), + .token = gh::gh_token(), + verbose = getOption("piggyback.verbose", default = TRUE)){ r <- parse_repo(repo) @@ -38,7 +40,7 @@ pb_releases <- function(repo = guess_repo(), .token = gh::gh_token(), verbose = c("!" = "No GitHub releases found for {.val {repo}}!", "You can make a new one with {.fun piggyback::pb_new_release}") )} - return(data.frame()) + return(invisible(data.frame())) } out <- data.frame( diff --git a/R/pb_new_release.R b/R/pb_new_release.R index 1aede56..2cc119c 100644 --- a/R/pb_new_release.R +++ b/R/pb_new_release.R @@ -17,8 +17,6 @@ #' identify the release as a pre-release. #' @inheritParams pb_upload #' @export -#' @importFrom jsonlite toJSON -#' @importFrom httr content GET POST stop_for_status #' @examples \dontrun{ #' pb_new_release("cboettig/piggyback-tests", "v0.0.5") #' } @@ -35,7 +33,7 @@ pb_new_release <- function(repo = guess_repo(), # if no releases exist, pb_releases returns a dataframe of releases if(nrow(releases) > 0 && tag %in% releases$tag_name){ - cli::cli_abort("Release tag {.val {tag}} already exists!") + cli::cli_abort("Release tag {.val {tag}} already exists!") } r <- parse_repo(repo) @@ -68,8 +66,10 @@ pb_new_release <- function(repo = guess_repo(), } ## Release info changed, so break caches - memoise::forget(pb_info) - memoise::forget(pb_releases) + try({ + memoise::forget(pb_info) + memoise::forget(pb_releases) + }) release <- httr::content(resp) cli::cli_alert_success() diff --git a/R/pb_upload.R b/R/pb_upload.R index 8eb1f84..e7dd7c5 100644 --- a/R/pb_upload.R +++ b/R/pb_upload.R @@ -144,7 +144,7 @@ pb_upload_file <- function(file, no_update <- local_timestamp <= df[i, "timestamp"] if (no_update) { cli::cli_warn("Matching or more recent version of {.file {file_path}} found on GH, not uploading.") - return(NULL) + return(invisible(NULL)) } } @@ -158,7 +158,7 @@ pb_upload_file <- function(file, ) } else { cli::cli_warn("Skipping upload of {.file {df$file_name[i]}} as file exists on GitHub and {.code overwrite = FALSE}") - return(NULL) + return(invisible(NULL)) } } @@ -172,9 +172,9 @@ pb_upload_file <- function(file, cat("\n") - if(getOption("piggyback.verbose",default = TRUE)) httr::warn_for_status(r) + if(getOption("piggyback.verbose", default = TRUE)) httr::warn_for_status(r) ## Release info changed, so break cache - memoise::forget(pb_info) + try({memoise::forget(pb_info)}) invisible(r) } diff --git a/R/piggyback.R b/R/piggyback.R index 370fa96..3aa43d1 100644 --- a/R/piggyback.R +++ b/R/piggyback.R @@ -11,9 +11,4 @@ #' and downloaded programmatically from scripts. No authentication is required to #' download data from public repositories. #' -#' It has two main modes or workflows: -#' -#' - [pb_upload()] / [pb_download()]: Upload and download individual files to/from -#' the desired release of the specified repository -#' "_PACKAGE" diff --git a/R/utils.R b/R/utils.R index a9c030e..6c65113 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,3 +1,26 @@ +#' Remove null elements of a list +#' @keywords internal +#' @noRd +compact <- function (l) Filter(Negate(is.null), l) + +#' trycatch/try wrapper +#' @keywords internal +#' @noRd +maybe <- function(expr, otherwise, quiet = TRUE) { + if (missing(otherwise)) { + try(expr, silent = quiet) + } else { + tryCatch(expr, + error = function(e) { + if (!quiet) { + message("Error: ", e$message) + } + otherwise + } + ) + } +} + #' Parses repository spec and errors if it fails #' @keywords internal #' @noRd @@ -13,91 +36,38 @@ parse_repo <- function(repo){ return(r) } -#' Remove null elements of a list + +#' Guesses GH repo based on git remote info for current git directory #' @keywords internal #' @noRd -compact <- function (l) Filter(Negate(is.null), l) +guess_repo <- function(path = ".") { + + exists <- requireNamespace("gert", quietly = TRUE) + if(!exists) stop(paste( + "Install package 'gert' to let piggyback discover the", + "current repo, or provide your repo name explicitly")) -# Adapted from `usethis` and under -# GPL-3 (Copyright RStudio Inc) -# https://github.com/r-lib/usethis/blob/aacf687445c94d4f726e31f682308a9a3ef3f820/R/write.R -# See https://github.com/r-lib/usethis/issues/366 -# write_union <- function(base_path, path, new_lines, quiet = FALSE) { -# stopifnot(is.character(new_lines)) -# -# full_path <- file.path(base_path, path) -# if (file.exists(full_path)) { -# lines <- readLines(full_path, warn = FALSE) -# } else { -# lines <- character() -# } -# -# new <- setdiff(new_lines, lines) -# if (length(new) == 0) { -# return(invisible(FALSE)) -# } -# -# if (!quiet) { -# quoted <- paste0(value(new), collapse = ", ") -# done("Adding ", quoted, " to ", value(path)) -# } -# -# all <- union(lines, new_lines) -# write_utf8(full_path, all) -# } -# -# write_utf8 <- function(path, lines) { -# stopifnot(is.character(path)) -# stopifnot(is.character(lines)) -# -# con <- file(path, encoding = "utf-8") -# on.exit(close(con), add = TRUE) -# -# if (length(lines) > 1) { -# lines <- paste0(lines, "\n", collapse = "") -# } -# cat(lines, file = con, sep = "") -# -# invisible(TRUE) -# } -# -# done <- function(...) { -# bullet(paste0(...), bullet = crayon::green(clisymbols::symbol$tick)) -# } -# -# value <- function(...) { -# x <- paste0(...) -# crayon::blue(encodeString(x, quote = "'")) -# } -# -# bullet <- function(lines, bullet) { -# lines <- paste0(bullet, " ", lines) -# cat_line(lines) -# } -# -# cat_line <- function(...) { -# cat(..., "\n", sep = "") -# } + repo <- gert::git_find(path) + remotes <- gert::git_remote_list(repo) + remotes_names <- remotes[["name"]] -# utils::askYesKnow is new to R 3.5.0; avoid using it for backwards compatibility -# askYesNo <- function(msg){ -# -# prompts <- c("Yes", "No", "Cancel") -# choices <- tolower(prompts) -# msg1 <- paste0("(", paste(choices, collapse = "/"), ") ") -# -# if (nchar(paste0(msg, msg1)) > 250) { -# cat(msg, "\n") -# msg <- msg1 -# } -# else msg <- paste0(msg, " ", msg1) -# -# ans <- readline(msg) -# match <- pmatch(tolower(ans), tolower(choices)) -# -# if (!nchar(ans)) -# TRUE -# else if (is.na(match)) -# stop("Unrecognized response ", dQuote(ans)) -# else c(TRUE, FALSE, NA)[match] -# } + # When there are more than 1 remote, we prefer "upstream" + # then "origin." If neither exists, we error to avoid + # ambiguity. + remote <- if (length(remotes_names) > 1) { + if ("upstream" %in% remotes_names) { + "upstream" + } else if ("origin" %in% remotes_names) { + "origin" + } else + stop("Cannot infer repo, please provide `repo` explicitly.", + call. = FALSE) + } else { + remotes_names + } + + addr <- remotes[remotes[["name"]] == remote, "url"][["url"]] + + out <- gsub(".*[:|/]([^/]+/[^/]+)(?:\\.git$)?", "\\1", addr) + gsub("\\.git$", "", out) +} diff --git a/R/zzz.R b/R/zzz.R index 1e025a6..41022fc 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,5 +1,5 @@ .onLoad <- function(libname,pkgname){ - cache_duration <- as.numeric(Sys.getenv("piggyback_cache_duration",unset = "600")) + cache_duration <- as.numeric(Sys.getenv("piggyback_cache_duration", unset = "600")) assign(x = "pb_info", value = memoise::memoise(pb_info, ~ memoise::timeout(cache_duration)), envir = parent.env(environment())) diff --git a/README.Rmd b/README.Rmd index 09ef4e7..707ca37 100644 --- a/README.Rmd +++ b/README.Rmd @@ -44,7 +44,7 @@ Install from CRAN via install.packages("piggyback") ``` -You can install the development version from [GitHub](https://github.com/) with: +You can install the development version from [GitHub](https://github.com/ropensci/piggyback) with: ``` r # install.packages("devtools") @@ -75,12 +75,8 @@ readr::write_tsv(mtcars, "mtcars.tsv.gz") pb_upload("mtcars.tsv.gz", repo = "cboettig/piggyback-tests") ``` - - - ## Git LFS and other alternatives - `piggyback` acts like a poor soul's [Git LFS](https://git-lfs.github.com/). Git LFS is not only expensive, it also [breaks GitHub's collaborative model](https://angryfrenchman.org/github-s-large-file-storage-is-no-panacea-for-open-source-quite-the-opposite-12c0e16a9a91) -- basically if someone wants to submit a PR with a simple edit to your docs, they cannot fork your repository since that would otherwise count against your Git LFS storage. Unlike Git LFS, `piggyback` doesn't take over your standard `git` client, it just perches comfortably on the shoulders of your existing GitHub API. Data can be versioned by `piggyback`, but relative to `git LFS` versioning is less strict: uploads can be set as a new version or allowed to overwrite previously uploaded data. ## But what will GitHub think of this? diff --git a/README.md b/README.md index a2f457f..b45b061 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ install.packages("piggyback") ``` You can install the development version from -[GitHub](https://github.com/) with: +[GitHub](https://github.com/ropensci/piggyback) with: ``` r # install.packages("devtools") diff --git a/codemeta.json b/codemeta.json index ed68668..53d32c2 100644 --- a/codemeta.json +++ b/codemeta.json @@ -2,12 +2,12 @@ "@context": "https://doi.org/10.5063/schema/codemeta-2.0", "@type": "SoftwareSourceCode", "identifier": "piggyback", - "description": "Because larger (> 50 MB) data files cannot easily be committed to git,\n a different approach is required to manage data associated with an analysis in a \n GitHub repository. This package provides a simple work-around by allowing larger\n (up to 2 GB) data files to piggyback on a repository as assets attached to individual\n GitHub releases. These files are not handled by git in any way, but instead are\n uploaded, downloaded, or edited directly by calls through the GitHub API. These\n data files can be versioned manually by creating different releases. This approach\n works equally well with public or private repositories. Data can be uploaded\n and downloaded programmatically from scripts. No authentication is required to\n download data from public repositories.", + "description": "Because larger (> 50 MB) data files cannot easily be committed to git, a different approach is required to manage data associated with an analysis in a GitHub repository. This package provides a simple work-around by allowing larger (up to 2 GB) data files to piggyback on a repository as assets attached to individual GitHub releases. These files are not handled by git in any way, but instead are uploaded, downloaded, or edited directly by calls through the GitHub API. These data files can be versioned manually by creating different releases. This approach works equally well with public or private repositories. Data can be uploaded and downloaded programmatically from scripts. No authentication is required to download data from public repositories.", "name": "piggyback: Managing Larger Data on a GitHub Repository", "codeRepository": "https://github.com/ropensci/piggyback", "issueTracker": "https://github.com/ropensci/piggyback/issues", "license": "https://spdx.org/licenses/GPL-3.0", - "version": "0.1.1", + "version": "0.1.1.9002", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", @@ -47,6 +47,12 @@ "givenName": "Kevin", "familyName": "Kuo", "@id": "https://orcid.org/0000-0001-7803-7901" + }, + { + "@type": "Person", + "givenName": "Tan", + "familyName": "Ho", + "@id": "https://orcid.org/0000-0001-8388-5155" } ], "copyrightHolder": [ @@ -185,75 +191,75 @@ "softwareRequirements": { "1": { "@type": "SoftwareApplication", - "identifier": "gh", - "name": "gh", + "identifier": "cli", + "name": "cli", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=gh" + "sameAs": "https://CRAN.R-project.org/package=cli" }, "2": { "@type": "SoftwareApplication", - "identifier": "httr", - "name": "httr", + "identifier": "glue", + "name": "glue", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=httr" + "sameAs": "https://CRAN.R-project.org/package=glue" }, "3": { "@type": "SoftwareApplication", - "identifier": "jsonlite", - "name": "jsonlite", + "identifier": "gh", + "name": "gh", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=jsonlite" + "sameAs": "https://CRAN.R-project.org/package=gh" }, "4": { "@type": "SoftwareApplication", - "identifier": "fs", - "name": "fs", + "identifier": "httr", + "name": "httr", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=fs" + "sameAs": "https://CRAN.R-project.org/package=httr" }, "5": { "@type": "SoftwareApplication", - "identifier": "crayon", - "name": "crayon", + "identifier": "jsonlite", + "name": "jsonlite", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=crayon" + "sameAs": "https://CRAN.R-project.org/package=jsonlite" }, "6": { "@type": "SoftwareApplication", - "identifier": "clisymbols", - "name": "clisymbols", + "identifier": "fs", + "name": "fs", "provider": { "@id": "https://cran.r-project.org", "@type": "Organization", "name": "Comprehensive R Archive Network (CRAN)", "url": "https://cran.r-project.org" }, - "sameAs": "https://CRAN.R-project.org/package=clisymbols" + "sameAs": "https://CRAN.R-project.org/package=fs" }, "7": { "@type": "SoftwareApplication", @@ -281,5 +287,5 @@ }, "SystemRequirements": null }, - "fileSize": "0KB" + "fileSize": "357.526KB" } diff --git a/dev/examples/gh.R b/dev/examples/gh.R new file mode 100644 index 0000000..f657711 --- /dev/null +++ b/dev/examples/gh.R @@ -0,0 +1,105 @@ + +# asset_filename <- function(x, start = ".") { +# ## piggyback will no longer embed file structure in filename +# ## Asset uploading is simply flat storage +# x +# } + +# local_filename <- function(x) { +# # x <- gsub("^manifest.json$", ".manifest.json", x) +# gsub("\\.2f", .Platform$file.sep, x) +# } + + +##################### Generic helpers ################## +# api_error_msg <- function(r) { +# cli::cli_warn( +# c("!"="Cannot access release data for repo {.val {paste0(r[[1]], '/', r[[2]])}}.", +# "Check that you have provided a {code .token} and that there is at least one release on your repo" +# )) +# } + + +# Adapted from `usethis` and under +# GPL-3 (Copyright RStudio Inc) +# https://github.com/r-lib/usethis/blob/aacf687445c94d4f726e31f682308a9a3ef3f820/R/write.R +# See https://github.com/r-lib/usethis/issues/366 +# write_union <- function(base_path, path, new_lines, quiet = FALSE) { +# stopifnot(is.character(new_lines)) +# +# full_path <- file.path(base_path, path) +# if (file.exists(full_path)) { +# lines <- readLines(full_path, warn = FALSE) +# } else { +# lines <- character() +# } +# +# new <- setdiff(new_lines, lines) +# if (length(new) == 0) { +# return(invisible(FALSE)) +# } +# +# if (!quiet) { +# quoted <- paste0(value(new), collapse = ", ") +# done("Adding ", quoted, " to ", value(path)) +# } +# +# all <- union(lines, new_lines) +# write_utf8(full_path, all) +# } +# +# write_utf8 <- function(path, lines) { +# stopifnot(is.character(path)) +# stopifnot(is.character(lines)) +# +# con <- file(path, encoding = "utf-8") +# on.exit(close(con), add = TRUE) +# +# if (length(lines) > 1) { +# lines <- paste0(lines, "\n", collapse = "") +# } +# cat(lines, file = con, sep = "") +# +# invisible(TRUE) +# } +# +# done <- function(...) { +# bullet(paste0(...), bullet = crayon::green(clisymbols::symbol$tick)) +# } +# +# value <- function(...) { +# x <- paste0(...) +# crayon::blue(encodeString(x, quote = "'")) +# } +# +# bullet <- function(lines, bullet) { +# lines <- paste0(bullet, " ", lines) +# cat_line(lines) +# } +# +# cat_line <- function(...) { +# cat(..., "\n", sep = "") +# } + +# utils::askYesKnow is new to R 3.5.0; avoid using it for backwards compatibility +# askYesNo <- function(msg){ +# +# prompts <- c("Yes", "No", "Cancel") +# choices <- tolower(prompts) +# msg1 <- paste0("(", paste(choices, collapse = "/"), ") ") +# +# if (nchar(paste0(msg, msg1)) > 250) { +# cat(msg, "\n") +# msg <- msg1 +# } +# else msg <- paste0(msg, " ", msg1) +# +# ans <- readline(msg) +# match <- pmatch(tolower(ans), tolower(choices)) +# +# if (!nchar(ans)) +# TRUE +# else if (is.na(match)) +# stop("Unrecognized response ", dQuote(ans)) +# else c(TRUE, FALSE, NA)[match] +# } diff --git a/man/pb_releases.Rd b/man/pb_releases.Rd index 6e5de01..b640be5 100644 --- a/man/pb_releases.Rd +++ b/man/pb_releases.Rd @@ -4,7 +4,11 @@ \alias{pb_releases} \title{List releases in repository} \usage{ -pb_releases(repo = guess_repo(), .token = gh::gh_token(), verbose = TRUE) +pb_releases( + repo = guess_repo(), + .token = gh::gh_token(), + verbose = getOption("piggyback.verbose", default = TRUE) +) } \arguments{ \item{repo}{GitHub repository specification in the form of \code{"owner/repo"}, if not specified will try to guess repo based on current working directory.} diff --git a/man/piggyback-package.Rd b/man/piggyback-package.Rd index 22088a7..4e07fd4 100644 --- a/man/piggyback-package.Rd +++ b/man/piggyback-package.Rd @@ -17,13 +17,6 @@ works equally well with public or private repositories. Data can be uploaded and downloaded programmatically from scripts. No authentication is required to download data from public repositories. } -\details{ -It has two main modes or workflows: -\itemize{ -\item \code{\link[=pb_upload]{pb_upload()}} / \code{\link[=pb_download]{pb_download()}}: Upload and download individual files to/from -the desired release of the specified repository -} -} \seealso{ Useful links: \itemize{ diff --git a/vignettes/alternatives.Rmd b/vignettes/alternatives.Rmd index 022144c..65af3da 100644 --- a/vignettes/alternatives.Rmd +++ b/vignettes/alternatives.Rmd @@ -49,7 +49,3 @@ In terms of practical implementation, `datastorr` also creates a new release eve Another creative solution (hack), at least for some file types, is to break large files into multiple smaller files, and commit those to one or many GitHub repositories. While [sharding](https://en.wikipedia.org/wiki/Shard_(database_architecture)) is sometimes a legitimate strategy, it has many obvious practical disadvantages and limitations. - - - - diff --git a/vignettes/intro.Rmd b/vignettes/intro.Rmd index 2e35932..05b84c3 100644 --- a/vignettes/intro.Rmd +++ b/vignettes/intro.Rmd @@ -160,7 +160,7 @@ Similarly, you can download all current data assets of the latest or specified r ## Caching -To reduce API calls to GitHub, piggyback caches most calls with a timeout of 1 second by default. This avoids repeating identical requests to update it's internal record of the repository data (releases, assets, timestamps, etc) during programmatic use. You can increase or decrease this delay by setting the environmental variable in seconds, e.g. `Sys.setenv("piggyback_cache_duration"=10)` for a longer delay or `Sys.setenv("piggyback_cache_duration"=0)` to disable caching. +To reduce API calls to GitHub, piggyback caches most calls with a timeout of 1 second by default. This avoids repeating identical requests to update it's internal record of the repository data (releases, assets, timestamps, etc) during programmatic use. You can increase or decrease this delay by setting the environmental variable in seconds, e.g. `Sys.setenv("piggyback_cache_duration"=10)` for a longer delay or `Sys.setenv("piggyback_cache_duration"=0)` to disable caching, and then restarting R. ## Valid file names From 13fd47848d6daa035d4a302cdb52239bf1c7f6a4 Mon Sep 17 00:00:00 2001 From: Tan-DESKPC Date: Mon, 25 Apr 2022 23:37:48 -0400 Subject: [PATCH 29/29] fixup success alert for pb_new_release --- R/pb_new_release.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pb_new_release.R b/R/pb_new_release.R index 2cc119c..882c027 100644 --- a/R/pb_new_release.R +++ b/R/pb_new_release.R @@ -72,6 +72,6 @@ pb_new_release <- function(repo = guess_repo(), }) release <- httr::content(resp) - cli::cli_alert_success() + cli::cli_alert_success("Created new release {.val {name}}.") invisible(release) }