Skip to content

Commit

Permalink
Refactor paginate_req_perform()
Browse files Browse the repository at this point in the history
  • Loading branch information
mgirlich committed Sep 19, 2023
1 parent 0b1394f commit 34b0855
Showing 1 changed file with 21 additions and 24 deletions.
45 changes: 21 additions & 24 deletions R/paginate.R
Original file line number Diff line number Diff line change
Expand Up @@ -102,51 +102,48 @@ paginate_req_perform <- function(req,
check_has_pagination_policy(req)
check_number_whole(max_pages, allow_infinite = TRUE, min = 1)

resp <- req_perform(req)
parse_resp <- req$policies$paginate$parse_resp
parsed <- parse_resp(resp)

f_n_pages <- req$policies$paginate$n_pages

n_pages <- min(f_n_pages(resp, parsed), max_pages)
# the implementation below doesn't really support an infinite amount of pages
# but 100e3 should be plenty
n_pages <- max_pages
if (is.infinite(n_pages)) {
n_pages <- 100e3
}

pb <- create_progress_bar(
total = n_pages,
name = "Paginate",
config = progress
)
show_progress <- !is.null(pb)

# the implementation below doesn't really support an infinite amount of pages
# but 100e3 should be plenty
if (is.infinite(n_pages)) {
n_pages <- 100e3
}

out <- vector("list", length = n_pages)
out[[1]] <- parsed

for (page in seq2(2, n_pages)) {
req <- paginate_next_request(resp, req, parsed)
if (is.null(req)) {
page <- page - 1L
break
}
page <- 0L
while ((page + 1) <= n_pages) {
page <- page + 1L

resp <- req_perform(req)
parsed <- parse_resp(resp)

if (page == 1) {
n_pages <- min(f_n_pages(resp, parsed), n_pages)
}

out[[page]] <- parsed
if (show_progress) cli::cli_progress_update(total = n_pages)

if (show_progress) cli::cli_progress_update()
req <- paginate_next_request(resp, req, parsed)
if (is.null(req)) {
break
}
}
if (show_progress) cli::cli_progress_done()

# `page` may be `NULL` if `start >= n_pages`
page <- page %||% 1L

# remove unused end of `out` in case the pagination loop exits before all
# `max_pages` is reached
if (page < n_pages) {
# remove unused tail of `out`
if (page < length(out)) {
out <- out[seq2(1, page)]
}

Expand Down

0 comments on commit 34b0855

Please sign in to comment.