r-box · ijlyttle · Jul 13, 2018 · Jun 18, 2018 · Jun 20, 2018 · Jun 20, 2018
diff --git a/NEWS.md b/NEWS.md
@@ -2,13 +2,17 @@
 
 ## Improvements
 
+* converts pagination method from offset to marker-based paging (#79, @awong234)
+
 * adds option to return specified `fields` in `box_ls()` (#72, @awong234)
 
 * adds updated screen shots (pngs in `images/`) and description for creating a Box App with the new Box Developer Console UI. Also added reference for `box_auth_on_attach` in step 4 'And you're done'. (#57, @nathancday)
 
 ## Bug Fixes
 
-* fixes bug in `box_pagination()` to enforce integer for offset (#71, @awong234)
+* `box_pagination()` refactored to employ marker-based paging instead of offset-based paging, avoiding a hard limit of 300,000 offset (#74, @awong234)
+
+* ~~fixes bug in `box_pagination()` to enforce integer for offset (#71, @awong234)~~ Deprecated, pagination now uses marker-based paging.
 
 * fixes bug in `box_search_files()` (#61, @j450h1)
 

diff --git a/R/boxr_misc.R b/R/boxr_misc.R
@@ -1,36 +1,39 @@
 #' Obtain a data.frame describing the contents of a box.com folder
-#' 
+#'
 #' @param dir_id The box.com id for the folder that you'd like to query
 #' @param limit  Maximum number of entries to retrieve per query-page
 #' @param max    Maximum number of entries to retrieve in total
-#' @param fields Specify what fields to query as a character vector. 
-#'   The default value NULL will return all possible columns: 
-#'  `"modified_at"`, `"content_modified_at"`, `"name"`, `"id"`, `"type"`, 
-#'  `"sha1"` ,`"size"`, `"owned_by"`, `"path_collection"`, `"description"`
-#' 
-#' @return A data.frame describing the contents of the the folder specified by 
+#' @param fields Specify what fields to query as a character vector. The default
+#'   value NULL will return all possible columns: `"modified_at"`,
+#'   `"content_modified_at"`, `"name"`, `"id"`, `"type"`, `"sha1"` ,`"size"`,
+#'   `"owned_by"`, `"path_collection"`, `"description"`
+#'
+#' @return A data.frame describing the contents of the the folder specified by
 #'   `dir_id`. Non recursive.
-#'   
-#' @author Brendan Rocks \email{foss@@brendanrocks.com} and Ian Lyttle 
-#'   \email{ian.lyttle@@schneider-electric.com}
-#'   
-#' @seealso [box_fetch()] and [box_push()] for synchronizing
-#'   the contents of local and remote directories. [list.files()] for
-#'   examining the contents of local directories.
-#'   
+#'
+#' @author Brendan Rocks \email{foss@@brendanrocks.com}, Ian Lyttle
+#'   \email{ian.lyttle@@schneider-electric.com}, and Alec Wong \email{[email protected]}
+#'
+#' @seealso [box_fetch()] and [box_push()] for synchronizing the contents of
+#'   local and remote directories. [list.files()] for examining the contents of
+#'   local directories.
+#'
 #' @export
 box_ls <- function(dir_id = box_getwd(), limit = 100, max = Inf, fields = NULL) {
 
-  # maybe some logic here to check that limit <= 1000
+  if (limit > 1000) {
+    warning("The maximum limit is 1000; box_ls is using 1000.")
+    limit <- 1000
+  }
 
   checkAuth()
-
+    
   url_root <- "https://api.box.com/2.0"
 
   url <- httr::parse_url(
     paste(url_root, "folders", box_id(dir_id), "items", sep = "/")
   )
-
+  
   fields_all <- 
     c("modified_at" ,"content_modified_at", "name", "id", "type",
       "sha1" ,"size", "owned_by", "path_collection", "description")
@@ -49,55 +52,57 @@ box_ls <- function(dir_id = box_getwd(), limit = 100, max = Inf, fields = NULL)
     limit = limit
   )
 
-  out <- box_pagination(url, max = max)
+  out <- box_pagination(url = url, max = max)
 
   class(out) <- "boxr_object_list"
   return(out)
 }
 
 
 #' @keywords internal
-box_pagination <- function(url, max = 200) {
-
-  out       <- list()
+box_pagination <- function(url, max){
+
+  marker <- character(0)
+  n_so_far <- 0
+  out <- list()
+  url$query$usemarker <- TRUE
   next_page <- TRUE
-  page      <- 1
-  n_so_far  <- 0
 
   while (next_page) {
-
-    limit <- url$query$limit
-
-    url$query$offset <- as.integer((page - 1) * limit)
-
-    req      <- httr::GET(
-      url, 
+
+    req <- httr::GET(
+      url,
       httr::config(token = getOption("boxr.token"))
-    )
-    
+    )    
+
     if (req$status_code == 404) {
-      message("box.com indicates that no results were found")
+      message("Error 404: box.com indicates that no results were found, or the folder specified does not exist in your account.")
       return()
-    }
-    
+    }    
+
     httr::stop_for_status(req)
-
-    resp     <- httr::content(req)
+    resp <- httr::content(req)
     n_req    <- length(resp$entries)
     n_so_far <- n_so_far + n_req
-    total    <- resp$total_count
-
-    if (!n_so_far < total | n_so_far >= max)
+    out <- c(out, resp$entries)
+    marker <- resp$next_marker
+
+    if (is.null(marker)) {
       next_page <- FALSE
+    } else {
+        url$query$marker <- marker
+    }
+
+    if (n_so_far >= max) {
+      return(out)
+    }
 
-    out     <- c(out, resp$entries)
-    page    <- page + 1
   }
 
   return(out)
+
 }
 
-
 #' Get/Set Default box.com directory/folder
 #' 
 #' @description Providing analgous functionality for the jbase **`R`**

diff --git a/man/box_ls.Rd b/man/box_ls.Rd