feat(empty-results): support empty non-terminal pages

BREAKING CHANGE: Data sources must return a final cursor value
for the result set.

__tests__/LetterDataSource.ts

@@ -11,10 +11,12 @@ export class MockLetterSource implements DataSource<LetterResult> {
 
   async getNextResults(cursor: string | undefined) {
     const startIndex = this.results.findIndex(result => result.cursor > (cursor ?? ''));
+    const results = startIndex === -1 ? [] : this.results.slice(startIndex, startIndex + this.pageSize);
     return {
       total: this.results.length,
       hasMore: startIndex + this.pageSize < this.results.length,
-      results: startIndex === -1 ? [] : this.results.slice(startIndex, startIndex + this.pageSize),
+      results,
+      cursor: results[results.length - 1].cursor,
     };
   }
 

src/asDataGenerator.ts

@@ -14,18 +14,17 @@ export function asDataGenerator<T extends ResultWithCursor>(
       while (continueFetching) {
         const response = await dataSource.getNextResults(currentCursor);
         total = response.total;
-        if (response.results.length === 0) {
-          continueFetching = false;
-        } else {
-          for (const result of response.results) {
-            if (!filter || filter(result)) {
-              yield result;
-            }
-            currentCursor = result.cursor; // Assuming the cursor for the next fetch is from the last item
+        continueFetching = response.hasMore;
+
+        for (const result of response.results || []) {
+          if (!filter || filter(result)) {
+            yield result;
           }
-          // Continue fetching only if we received as many results as we asked for, implying there might be more.
-          continueFetching = response.hasMore;
+          currentCursor = result.cursor; // Assuming the cursor for the next fetch is from the last item
         }
+        // There is a nuance here - we COULD return this result-set cursor for the last
+        // result in the yield above, and that would avoid fetching unused results next time.
+        currentCursor = response.cursor;
       }
     },
     sortKey: dataSource.sortKey,

src/types.ts

@@ -13,6 +13,10 @@ export interface DatasourceResults<T extends ResultWithCursor> {
   // If total is available from ALL data sources, a total will
   // be returned from the paged set.
   total?: number;
+  // There can be cases where you don't return results, but the cursor
+  // has advanced anyhow. Once we run out of results, we will use this
+  // cursor.
+  cursor: string;
 }
 
 /**