tar_resources_custom_format()

DESCRIPTION

@@ -12,7 +12,7 @@ Description: Pipeline tools coordinate the pieces of computationally
   The methodology in this package
   borrows from GNU 'Make' (2015, ISBN:978-9881443519)
   and 'drake' (2018, <doi:10.21105/joss.00550>).
-Version: 1.6.0.9001
+Version: 1.6.0.9002
 License: MIT + file LICENSE
 URL: https://docs.ropensci.org/targets/, https://github.com/ropensci/targets
 BugReports: https://github.com/ropensci/targets/issues

NAMESPACE

@@ -15,6 +15,7 @@ S3method(print,tar_pipeline)
 S3method(print,tar_resources_aws)
 S3method(print,tar_resources_clustermq)
 S3method(print,tar_resources_crew)
+S3method(print,tar_resources_custom_format)
 S3method(print,tar_resources_feather)
 S3method(print,tar_resources_fst)
 S3method(print,tar_resources_future)
@@ -28,6 +29,7 @@ S3method(resources_validate,default)
 S3method(resources_validate,tar_resources_aws)
 S3method(resources_validate,tar_resources_clustermq)
 S3method(resources_validate,tar_resources_crew)
+S3method(resources_validate,tar_resources_custom_format)
 S3method(resources_validate,tar_resources_feather)
 S3method(resources_validate,tar_resources_fst)
 S3method(resources_validate,tar_resources_future)
@@ -466,6 +468,7 @@ export(tar_resources)
 export(tar_resources_aws)
 export(tar_resources_clustermq)
 export(tar_resources_crew)
+export(tar_resources_custom_format)
 export(tar_resources_feather)
 export(tar_resources_fst)
 export(tar_resources_future)

NEWS.md

@@ -1,4 +1,4 @@
-# targets 1.6.0.9001
+# targets 1.6.0.9002
 
 ## Invalidating changes
 
@@ -8,6 +8,7 @@
 
 * Inform and prompt the user when the pipeline was built with an old version of `targets` and changes to the package will cause the current work to rerun (#1244). For the `tar_make*()` functions, `utils::menu()` prompts the user to give people a chance to downgrade if necessary.
 * For type safety in the internal database class, read all columns as character vectors in `data.table::fread()`, then convert them to the correct types afterwards.
+* Add a new `tar_resources_custom_format()` function which can pass environment variables to customize the behavior of custom `tar_format()` storage formats (#1263, #1232, @Aariq, @noamross).
 
 # targets 1.6.0
 

R/class_resources_custom_format.R

@@ -0,0 +1,31 @@
+resources_custom_format_init <- function(
+  envvars = NULL
+) {
+  resources_custom_format_new(
+    envvars = envvars
+  )
+}
+
+resources_custom_format_new <- function(
+  envvars = NULL
+) {
+  force(envvars)
+  enclass(environment(), c("tar_resources_custom_format", "tar_resources"))
+}
+
+#' @export
+resources_validate.tar_resources_custom_format <- function(resources) {
+  if (!is.null(resources$envvars)) {
+    tar_assert_chr(resources$envvars)
+    tar_assert_none_na(resources$envvars)
+    tar_assert_named(resources$controller)
+  }
+}
+
+#' @export
+print.tar_resources_custom_format <- function(x, ...) {
+  cat(
+    "<tar_resources_custom_format>\n ",
+    paste0(paste_list(as.list(x)), collapse = "\n  ")
+  )
+}

R/class_store_custom.R

@@ -81,6 +81,7 @@ store_assert_format_setting.format_custom <- function(format) {
 #' @export
 store_read_path.tar_store_custom <- function(store, path) {
   store_custom_call_method(
+    store = store,
     text = store$read,
     args = list(path = path)
   )
@@ -89,6 +90,7 @@ store_read_path.tar_store_custom <- function(store, path) {
 #' @export
 store_write_path.tar_store_custom <- function(store, object, path) {
   store_custom_call_method(
+    store = store,
     text = store$write,
     args = list(object = object, path = path)
   )
@@ -97,6 +99,7 @@ store_write_path.tar_store_custom <- function(store, object, path) {
 #' @export
 store_marshal_object.tar_store_custom <- function(store, object) {
   store_custom_call_method(
+    store = store,
     text = store$marshal,
     args = list(object = object)
   )
@@ -105,6 +108,7 @@ store_marshal_object.tar_store_custom <- function(store, object) {
 #' @export
 store_unmarshal_object.tar_store_custom <- function(store, object) {
   store_custom_call_method(
+    store = store,
     text = store$unmarshal,
     args = list(object = object)
   )
@@ -113,6 +117,7 @@ store_unmarshal_object.tar_store_custom <- function(store, object) {
 #' @export
 store_convert_object.tar_store_custom <- function(store, object) {
   store_custom_call_method(
+    store = store,
     text = store$convert,
     args = list(object = object)
   )
@@ -121,12 +126,20 @@ store_convert_object.tar_store_custom <- function(store, object) {
 #' @export
 store_copy_object.tar_store_custom <- function(store, object) {
   store_custom_call_method(
+    store = store,
     text = store$copy,
     args = list(object = object)
   )
 }
 
-store_custom_call_method <- function(text, args) {
+store_custom_call_method <- function(store, text, args) {
+  envvars <- store$resources$custom_format$envvars
+  if (length(envvars)) {
+    names <- names(envvars)
+    previous <- Sys.getenv(names, names = TRUE)
+    on.exit(do.call(what = Sys.setenv, args = as.list(previous)))
+    do.call(what = Sys.setenv, args = as.list(envvars))
+  }
   envir <- new.env(parent = baseenv())
   what <- eval(parse(text = text), envir = envir)
   do.call(what = what, args = args, envir = envir)

R/tar_resources.R

@@ -41,14 +41,16 @@
 #'   So the correct way to assign `clustermq` resources is through
 #'   [tar_option_set()], not [tar_target()]. `clustermq` resources
 #'   in individual [tar_target()] calls will be ignored.
-#' @param crew Output of function `tar_resources_crew()`
+#' @param crew Output of function [tar_resources_crew()]
 #'   with target-specific settings for integration with the
 #'   `crew` R package. These settings are arguments to the `push()`
 #'   method of the controller or controller group
 #'   object which control things like
 #'   auto-scaling behavior and the controller to use in the case
 #'   of a controller group.
-#' @param feather Output of function `tar_resources_feather()`.
+#' @param custom_format Output of function [tar_resources_custom_format()]
+#'   with configuration details for [tar_format()] storage formats.
+#' @param feather Output of function [tar_resources_feather()].
 #'   Non-default arguments to `arrow::read_feather()` and
 #'   `arrow::write_feather()` for `arrow`/feather-based storage formats.
 #'   Applies to all formats ending with the `"_feather"` suffix.
@@ -113,6 +115,7 @@ tar_resources <- function(
   aws = tar_option_get("resources")$aws,
   clustermq = tar_option_get("resources")$clustermq,
   crew = tar_option_get("resources")$crew,
+  custom_format = tar_option_get("resources")$custom_format,
   feather = tar_option_get("resources")$feather,
   fst = tar_option_get("resources")$fst,
   future = tar_option_get("resources")$future,

R/tar_resources_custom_format.R

@@ -0,0 +1,44 @@
+#' @title Target resources for custom storage formats
+#' @export
+#' @family resources
+#' @description Create the `custom_format` argument of `tar_resources()`
+#'   to specify optional target settings for custom storage formats.
+#' @details `tar_resources_custom_format()` accepts
+#'   target-specific settings to customize [tar_format()] storage formats.
+#' @inheritSection tar_resources Resources
+#' @return Object of class `"tar_resources_custom_format"`, to be supplied
+#'   to the `custom_format` argument of `tar_resources()`.
+#' @param envvars Named character vector of environment variables.
+#'   These environment variables are temporarily set just before each call to
+#'   the storage methods you define in [tar_format()]. Specific methods
+#'   like `read` can retrieve values from these environment variables
+#'   using `Sys.getenv()`. Set `envvars` to `NULL` to omit entirely.
+#' @examples
+#' # Somewhere in you target script file (usually _targets.R):
+#' tar_target(
+#'   name = target_name,
+#'   command = data.frame(x = 1),
+#'   format = tar_format(
+#'     read = function(path) {
+#'       readRDS(file = path)
+#'     },
+#'     write = function(object, path) {
+#'       version <- as.integer(Sys.getenv("SERIALIZATION", unset = "2"))
+#'       saveRDS(object = object, file = path, version = version)
+#'     }
+#'   ),
+#'   resources = tar_resources(
+#'     custom_format = tar_resources_custom_format(
+#'       envvars = c(SERIALIZATION = "3")
+#'     )
+#'   )
+#' )
+tar_resources_custom_format <- function(
+  envvars = targets::tar_option_get("resources")$custom_format$envvars
+) {
+  out <- resources_custom_format_init(
+    envvars = envvars
+  )
+  resources_validate(out)
+  out
+}