transition to snapshot testing (#162)

tests/testthat/_snaps/autoplot.md

@@ -6,3 +6,11 @@
       Error in `autoplot()`:
       ! `type` must be one of "class" or "wflow_id", not "banana".
 
+# automatic selection of rank metric
+
+    Code
+      workflowsets:::pick_metric(two_class_res, "roc_auc", "accuracy")
+    Condition
+      Error in `halt()`:
+      ! Metric 'roc_auc' was not in the results.
+

tests/testthat/_snaps/comments.md

@@ -1,3 +1,75 @@
+# test comments
+
+    Code
+      two_class_set %>% comment_add("toe", "foot")
+    Condition
+      Error in `comment_add()`:
+      ! The 'id' value is not in wflow_id.
+
+---
+
+    Code
+      two_class_set %>% comment_add(letters, "foot")
+    Condition
+      Error in `comment_add()`:
+      ! `id` must be a single string, not a character vector.
+
+---
+
+    Code
+      two_class_set %>% comment_add(1:2, "foot")
+    Condition
+      Error in `comment_add()`:
+      ! `id` must be a single string, not an integer vector.
+
+---
+
+    Code
+      two_class_set %>% comment_add("none_cart", 1:2)
+    Condition
+      Error in `comment_add()`:
+      ! The comments should be character strings.
+
+---
+
+    Code
+      comments_1 %>% comment_add("none_cart", "Stuff.", append = FALSE)
+    Condition
+      Error in `comment_add()`:
+      ! There is already a comment for this id and `append = FALSE`.
+
+---
+
+    Code
+      comment_get(comments_1, id = letters)
+    Condition
+      Error in `comment_get()`:
+      ! 'id' should be a single character value.
+
+---
+
+    Code
+      comment_get(comments_1, id = letters[1])
+    Condition
+      Error in `comment_get()`:
+      ! The 'id' value is not in wflow_id.
+
+---
+
+    Code
+      comments_1 %>% comment_reset(letters)
+    Condition
+      Error in `comment_reset()`:
+      ! 'id' should be a single character value.
+
+---
+
+    Code
+      comments_1 %>% comment_reset("none_carts")
+    Condition
+      Error in `comment_reset()`:
+      ! The 'id' value is not in wflow_id.
+
 # print comments
 
     Code

tests/testthat/_snaps/extract.md

@@ -1,5 +1,41 @@
 # extracts
 
+    Code
+      extract_fit_engine(car_set_1, id = "reg_lm")
+    Condition
+      Error in `extract_fit_parsnip()`:
+      ! Can't extract a model fit from an untrained workflow.
+      i Do you need to call `fit()`?
+
+---
+
+    Code
+      extract_fit_parsnip(car_set_1, id = "reg_lm")
+    Condition
+      Error in `extract_fit_parsnip()`:
+      ! Can't extract a model fit from an untrained workflow.
+      i Do you need to call `fit()`?
+
+---
+
+    Code
+      extract_mold(car_set_1, id = "reg_lm")
+    Condition
+      Error in `extract_mold()`:
+      ! Can't extract a mold from an untrained workflow.
+      i Do you need to call `fit()`?
+
+---
+
+    Code
+      extract_recipe(car_set_1, id = "reg_lm")
+    Condition
+      Error in `extract_mold()`:
+      ! Can't extract a mold from an untrained workflow.
+      i Do you need to call `fit()`?
+
+---
+
     Code
       car_set_1 %>% extract_workflow_set_result("Gideon Nav")
     Condition
@@ -14,3 +50,11 @@
       Error in `halt()`:
       ! `id` must correspond to a single row in `x`.
 
+# extract single parameter from workflow set with untunable workflow
+
+    Code
+      hardhat::extract_parameter_dials(wf_set, id = "reg_lm", parameter = "non there")
+    Condition
+      Error in `extract_parameter_dials()`:
+      ! No parameter exists with id "non there".
+

tests/testthat/_snaps/leave-var-out-formulas.md

@@ -0,0 +1,16 @@
+# LOO var formulas
+
+    Code
+      leave_var_out_formulas(y ~ 1, data = form_data)
+    Condition
+      Error in `leave_var_out_formulas()`:
+      ! There should be at least 2 predictors in the formula.
+
+---
+
+    Code
+      leave_var_out_formulas(y ~ a, data = form_data)
+    Condition
+      Error in `leave_var_out_formulas()`:
+      ! There should be at least 2 predictors in the formula.
+

tests/testthat/_snaps/pull.md

@@ -0,0 +1,40 @@
+# pulling objects
+
+    Code
+      res <- car_set_1 %>% pull_workflow("reg_lm")
+    Condition
+      Warning:
+      `pull_workflow()` was deprecated in workflowsets 0.1.0.
+      i Please use `extract_workflow()` instead.
+
+---
+
+    Code
+      res <- car_set_1 %>% pull_workflow_set_result("reg_lm")
+    Condition
+      Warning:
+      `pull_workflow_set_result()` was deprecated in workflowsets 0.1.0.
+      i Please use `extract_workflow_set_result()` instead.
+
+---
+
+    Code
+      car_set_1 %>% pull_workflow_set_result("Gideon Nav")
+    Condition
+      Warning:
+      `pull_workflow_set_result()` was deprecated in workflowsets 0.1.0.
+      i Please use `extract_workflow_set_result()` instead.
+      Error in `halt()`:
+      ! No workflow ID found for 'Gideon Nav'
+
+---
+
+    Code
+      car_set_1 %>% pull_workflow("Coronabeth Tridentarius")
+    Condition
+      Warning:
+      `pull_workflow()` was deprecated in workflowsets 0.1.0.
+      i Please use `extract_workflow()` instead.
+      Error in `halt()`:
+      ! No workflow ID found for 'Coronabeth Tridentarius'
+

tests/testthat/_snaps/workflow-map.md

@@ -34,6 +34,18 @@
       i	No tuning parameters. `fit_resamples()` will be attempted
       i 3 of 3 resampling: nonlin_lm
 
+# failers
+
+    Code
+      res_loud <- car_set_3 %>% workflow_map(resamples = folds, seed = 2, verbose = TRUE,
+        grid = "a")
+    Message
+      i 1 of 2 tuning:     reg_knn
+      x 1 of 2 tuning:     reg_knn failed with: Error in check_grid(grid = grid, workflow = workflow, pset = pset) :   `grid` should be a positive integer or a data frame.
+      i	No tuning parameters. `fit_resamples()` will be attempted
+      i 2 of 2 resampling: reg_lm
+      v 2 of 2 resampling: reg_lm (ms)
+
 # fail informatively on mismatched spec/tuning function
 
     Code

tests/testthat/_snaps/workflow_set.md

@@ -50,6 +50,17 @@
     Output
       [1] "Error in `fit()`:\n! `col` must select a classed case weights column, as determined by `hardhat::is_case_weights()`. For example, it could be a column created by `hardhat::frequency_weights()` or `hardhat::importance_weights()`."
 
+# specifying an engine that does not allow case weights
+
+    Code
+      car_set_3 <- workflow_set(list(reg = mpg ~ ., nonlin = mpg ~ wt + 1 / sqrt(disp)),
+      list(lm = lr_spec, knn = knn_spec), case_weights = wts)
+    Condition
+      Warning:
+      Case weights are not enabled by the underlying model implementation for the following engine(s): kknn.
+      
+      The `case_weights` argument will be ignored for specifications using that engine.
+
 # specifying a case weight column that isn't in the resamples
 
     Code
@@ -112,6 +123,71 @@
     Output
       [1] "Error in `fit()`:\n! Can't select columns that don't exist.\nx Column `boop` doesn't exist."
 
+# crossing
+
+    Code
+      nrow(workflow_set(list(reg = mpg ~ ., nonlin = mpg ~ wt + 1 / sqrt(disp), two = mpg ~
+        wt + disp), list(lm = lr_spec, knn = knn_spec), cross = FALSE))
+    Condition
+      Error in `workflow_set()`:
+      ! The lengths of 'preproc' and 'models' are different and `cross = FALSE`.
+
+# checking resamples
+
+    Code
+      as_workflow_set(wt = f_1, disp = f_2)
+    Condition
+      Error in `halt()`:
+      ! Different resamples were used in the workflow 'result's. All elements of 'result' must use the same resamples.
+
+# constructor
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::select(-info))
+    Condition
+      Error in `halt()`:
+      ! The object should have columns: 'wflow_id', 'info', 'option', 'result'.
+
+---
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::mutate(info = "a"))
+    Condition
+      Error in `halt()`:
+      ! The 'info' column should be a list.
+
+---
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::mutate(result = "a"))
+    Condition
+      Error in `halt()`:
+      ! The 'result' column should be a list.
+
+---
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::mutate(option = "a"))
+    Condition
+      Error in `halt()`:
+      ! The 'option' column should be a list.
+
+---
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::mutate(wflow_id = 1))
+    Condition
+      Error in `halt()`:
+      ! The 'wflow_id' column should be character.
+
+---
+
+    Code
+      workflowsets:::new_workflow_set(car_set_1 %>% dplyr::mutate(wflow_id = "a"))
+    Condition
+      Error in `halt()`:
+      ! The 'wflow_id' column should contain unique, non-missing character strings.
+
 # pillar formatting
 
     # A workflow set/tibble: 3 x 4

tests/testthat/test-autoplot.R

@@ -109,8 +109,8 @@ test_that("automatic selection of rank metric", {
     workflowsets:::pick_metric(two_class_res, "roc_auc"),
     list(metric = "roc_auc", direction = "maximize")
   )
-  expect_error(
-    workflowsets:::pick_metric(two_class_res, "roc_auc", "accuracy"),
-    "was not in the results"
+  expect_snapshot(
+    error = TRUE,
+    workflowsets:::pick_metric(two_class_res, "roc_auc", "accuracy")
   )
 })

tests/testthat/test-collect-metrics.R

@@ -45,9 +45,9 @@ test_that("ranking models", {
       yj_trans_cart = 10, yj_trans_glm = 1, yj_trans_mars = 2
     )
 
-  expect_error(ranking_1 <- rank_results(two_class_res), regexp = NA)
+  expect_no_error(ranking_1 <- rank_results(two_class_res))
   expect_equal(nrow(ranking_1), sum(param_lines * 2))
 
-  expect_error(ranking_2 <- rank_results(two_class_res, select_best = TRUE), regexp = NA)
+  expect_no_error(ranking_2 <- rank_results(two_class_res, select_best = TRUE))
   expect_equal(nrow(ranking_2), nrow(two_class_res) * 2)
 })

tests/testthat/test-collect-predictions.R

@@ -79,33 +79,28 @@ check_prediction_results <- function(ind, x, summarize = FALSE, ...) {
 # ------------------------------------------------------------------------------
 
 test_that("collect predictions", {
-  expect_error(
-    res_car_set_1 <- collect_predictions(car_set_1),
-    regexp = NA
+  expect_no_error(
+    res_car_set_1 <- collect_predictions(car_set_1)
   )
   expect_true(nrow(mtcars) * nrow(car_set_1) == nrow(res_car_set_1))
 
-  expect_error(
-    res_car_set_2 <- collect_predictions(car_set_2),
-    regexp = NA
+  expect_no_error(
+    res_car_set_2 <- collect_predictions(car_set_2)
   )
   expect_true(nrow(mtcars) * nrow(car_set_2) == nrow(res_car_set_2))
 
-  expect_error(
-    res_car_set_2_reps <- collect_predictions(car_set_2, summarize = FALSE),
-    regexp = NA
+  expect_no_error(
+    res_car_set_2_reps <- collect_predictions(car_set_2, summarize = FALSE)
   )
   expect_true(nrow(mtcars) * nrow(car_set_2) * 2 == nrow(res_car_set_2_reps))
 
-  expect_error(
-    res_car_set_3 <- collect_predictions(car_set_3),
-    regexp = NA
+  expect_no_error(
+    res_car_set_3 <- collect_predictions(car_set_3)
   )
   expect_true(nrow(mtcars) * nrow(car_set_2) * 5 == nrow(res_car_set_3))
 
-  expect_error(
-    res_car_set_3_reps <- collect_predictions(car_set_3, summarize = FALSE),
-    regexp = NA
+  expect_no_error(
+    res_car_set_3_reps <- collect_predictions(car_set_3, summarize = FALSE)
   )
   expect_true(nrow(mtcars) * nrow(car_set_2) * 5 * 2 == nrow(res_car_set_3_reps))
 
@@ -151,17 +146,15 @@ test_that("dropping tuning parameter columns", {
     ignore.order = TRUE
   )
 
-  expect_error(
-    best_iter <- collect_predictions(car_set_3, select_best = TRUE, metric = "rmse"),
-    regexp = NA
+  expect_no_error(
+    best_iter <- collect_predictions(car_set_3, select_best = TRUE, metric = "rmse")
   )
   expect_true(
     nrow(dplyr::distinct(best_iter[, c(".config", "wflow_id")])) == 2
   )
-  expect_error(
+  expect_no_error(
     no_param <-
-      workflowsets:::select_bare_predictions(car_set_3$result[[1]], metric = "rmse", TRUE),
-    regex = NA
+      workflowsets:::select_bare_predictions(car_set_3$result[[1]], metric = "rmse", TRUE)
   )
   expect_named(
     no_param,