Patterns do not marshal properly

[wlandau]

The cause of #1264 and njtierney/geotargets#52 is actually a bug in targets. It turns out that aggregates of dynamic branches are unaware of marshaling. I am surprised this went unnoticed for so long. Anyway, here is a reprex with torch:

# _targets.R
library(targets)
tar_option_set(controller = crew::crew_controller_local())
list(
  tar_target(x, c(1L, 2L)),
  tar_target(
    y,
    torch::torch_tensor(x),
    pattern = map(x),
    format = "torch",
    iteration = "list",
    storage = "worker",
    retrieval = "worker"
  ),
  tar_target(
    z,
    y[[1]] + y[[2]],
    format = "torch",
    storage = "worker",
    retrieval = "main"
  )
)

# R console
> library(targets)
> tar_make()
▶ dispatched target x
● completed target x [0.005 seconds]
▶ dispatched branch y_5015dfa5778ec11d
● completed branch y_5015dfa5778ec11d [1.487 seconds]
▶ dispatched branch y_5ea9d15a285adf66
● completed branch y_5ea9d15a285adf66 [0 seconds]
● completed pattern y
▶ dispatched target z
✖ errored target z
✖ errored pipeline [5.649 seconds]
Error:
! Error running targets::tar_make()
Error messages: targets::tar_meta(fields = error, complete_only = TRUE)
Debugging guide: https://books.ropensci.org/targets/debugging.html
How to ask for help: https://books.ropensci.org/targets/help.html
Last error message:
    external pointer is not valid
Last error traceback:
    `+.torch_tensor`(y[[1]], y[[2]])
    torch_add(e1, e2)
    call_c_function(fun_name = "add", args = args, expected_types = expected...
    do_call(f, args)
    do.call(fun, args)
    (function (self, other, alpha)  {     .Call(`_torch_cpp_torch_namespace_...
    stop(list("external pointer is not valid", (function (self, other,      ...
    (function (condition)  {     state$error <- build_message(condition)    ...

[wlandau]

If storage is "main" in the last target, I get the original error reported:

# _targets.R
library(targets)
tar_option_set(controller = crew::crew_controller_local())
list(
  tar_target(x, c(1L, 2L)),
  tar_target(
    y,
    torch::torch_tensor(x),
    pattern = map(x),
    format = "torch",
    iteration = "list",
    storage = "worker",
    retrieval = "worker"
  ),
  tar_target(
    z,
    y[[1]] + y[[2]],
    format = "torch",
    storage = "main",
    retrieval = "main"
  )
)

# R console
> tar_make()
▶ dispatched target x
● completed target x [0.011 seconds]
▶ dispatched branch y_5015dfa5778ec11d
● completed branch y_5015dfa5778ec11d [0.943 seconds]
▶ dispatched branch y_5ea9d15a285adf66
● completed branch y_5ea9d15a285adf66 [0 seconds]
● completed pattern y
▶ dispatched target z
▶ ended pipeline [4.551 seconds]
Error:
! Error running targets::tar_make()
Error messages: targets::tar_meta(fields = error, complete_only = TRUE)
Debugging guide: https://books.ropensci.org/targets/debugging.html
How to ask for help: https://books.ropensci.org/targets/help.html
Last error message:
    target z error: no applicable method for 'torch_save' applied to an object of class "NULL"
Last error traceback:
    tryCatch(withCallingHandlers({ NULL saveRDS(do.call(do.call, c(readRDS("...
    tryCatchList(expr, classes, parentenv, handlers)
    tryCatchOne(tryCatchList(expr, names[-nh], parentenv, handlers[-nh]), na...
    doTryCatch(return(expr), name, parentenv, handler)
    tryCatchList(expr, names[-nh], parentenv, handlers[-nh])
    tryCatchOne(expr, names, parentenv, handlers[[1L]])
    doTryCatch(return(expr), name, parentenv, handler)
    withCallingHandlers({ NULL saveRDS(do.call(do.call, c(readRDS("/tmp/Rtmp...
    saveRDS(do.call(do.call, c(readRDS("/tmp/RtmpnrQQS4/callr-fun-8d1d70b5ac...
    do.call(do.call, c(readRDS("/tmp/RtmpnrQQS4/callr-fun-8d1d70b5ac59"), li...
    (function (what, args, quote = FALSE, envir = parent.frame()) { if (!is....
    (function (targets_function, targets_arguments, options, envir = NULL, s...
    tryCatch(out <- withCallingHandlers(targets::tar_callr_inner_try(targets...
    tryCatchList(expr, classes, parentenv, handlers)
    tryCatchOne(expr, names, parentenv, handlers[[1L]])
    doTryCatch(return(expr), name, parentenv, handler)
    withCallingHandlers(targets::tar_callr_inner_try(targets_function = targ...
    targets::tar_callr_inner_try(targets_function = targets_function, target...
    do.call(targets_function, targets_arguments)
    (function (pipeline, path_store, names_quosure, shortcut, reporter, seco...
    crew_init(pipeline = pipeline, meta = meta_init(path_store = path_store)...
    self$run_crew()
    self$iterate()
    self$conclude_worker_task()
    tar_assert_all_na(result$error, msg = paste("target", result$name, "erro...
    tar_throw_validate(msg %|||% default)
    tar_error(message = paste0(...), class = c("tar_condition_validate", "ta...
    rlang::abort(message = message, class = class, call = tar_empty_envir)
    signal_abort(cnd, .file)

[wlandau]

I think this is now fixed in https://github.com/ropensci/targets/tree/1266, but I have to run non-automated regression tests to confirm.

[wlandau]

Fixed in #1267