feat(filter): add a filtering data source function

src/FilteredDataSource.ts

@@ -0,0 +1,24 @@
+import { DataSource, ResultWithCursor } from './index';
+
+// For some data sources, local filtering is all that is possible, but it can be helpful to treat it
+// like a paged data source. SO, we will filter results after fetching them and then "top up" the datasource
+// with the next page of results if necessary. Suboptimal - to be clear.
+export function withFilter<T extends ResultWithCursor>(datasource: DataSource<T>, filter: (result: T) => boolean): DataSource<T> {
+  return {
+    async getResults(cursor: string | undefined, forward: boolean, limit: number) {
+      const results: T[] = [];
+      do {
+        // TODO we maybe should optimize the fetch size more
+        const nextResults = await datasource.getResults(results[results.length - 1]?.cursor || cursor, forward, limit);
+        results.push(...nextResults.results.filter(filter));
+        if (nextResults.results.length < limit) {
+          // Didn't get the required number of elements, must be the end
+          break;
+        }
+      } while (results.length < limit);
+      // Can't give a total count
+      return { results: results.slice(0, limit) };
+    },
+    sortKey: datasource.sortKey,
+  };
+}

src/index.ts

@@ -1,3 +1,4 @@
+export * from './FilteredDataSource';
 export * from './one-shot';
 export * from './stateful';
 export * from './types';

src/stateful.spec.ts

@@ -3,14 +3,15 @@ import { describe, it, expect } from 'vitest';
 import { MockLetterSource, mockDoubleLetters, mockLetters } from '../__tests__/LetterDataSource';
 
 import { statefulPager } from './stateful';
+import { withFilter } from './FilteredDataSource';
 
 // Comparator for sorting by cursor (ISO date)
 const comparator = (a: string, b: string) => a.localeCompare(b);
 
 describe('stateful pager', () => {
+  const dataSourceA = new MockLetterSource(mockLetters);
+  const dataSourceB = new MockLetterSource(mockDoubleLetters);
   it('should return the results in order', async () => {
-    const dataSourceA = new MockLetterSource(mockLetters);
-    const dataSourceB = new MockLetterSource(mockDoubleLetters);
 
     for (let i = 0; i < 5; i += 1) {
       const pager = await statefulPager({
@@ -107,4 +108,50 @@ describe('stateful pager', () => {
     `);
     }
   });
+
+  it.only('should work with filters', async () => {
+    const pager = await statefulPager({
+      comparator,
+    }, withFilter(dataSourceA, (r) => !!(r.data.length % 2)), withFilter(dataSourceB, (r) => !!(r.data.length % 2)));
+    const p1 = await pager.getNextResults(10);
+    expect(p1.results).toMatchInlineSnapshot(`
+      [
+        {
+          "cursor": "WyIyMDIzLTAxLTAxVDAwOjAwOjAwLjAwMFojQSJd",
+          "data": "A",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTAxVDAwOjAwOjAwLjAwMFojQSIsIjIwMjMtMDEtMDFUMDA6MDA6MDAuMDAwWiNBQUEiXQ==",
+          "data": "AAA",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTAyVDAwOjAwOjAwLjAwMFojQiIsIjIwMjMtMDEtMDFUMDA6MDA6MDAuMDAwWiNBQUEiXQ==",
+          "data": "B",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTAyVDAwOjAwOjAwLjAwMFojQiIsIjIwMjMtMDEtMDJUMDA6MDA6MDAuMDAwWiNCQkIiXQ==",
+          "data": "BBB",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTAzVDAwOjAwOjAwLjAwMFojQyIsIjIwMjMtMDEtMDJUMDA6MDA6MDAuMDAwWiNCQkIiXQ==",
+          "data": "C",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTA0VDAwOjAwOjAwLjAwMFojRCIsIjIwMjMtMDEtMDJUMDA6MDA6MDAuMDAwWiNCQkIiXQ==",
+          "data": "D",
+          "type": "letter",
+        },
+        {
+          "cursor": "WyIyMDIzLTAxLTA1VDAwOjAwOjAwLjAwMFojRSIsIjIwMjMtMDEtMDJUMDA6MDA6MDAuMDAwWiNCQkIiXQ==",
+          "data": "E",
+          "type": "letter",
+        },
+      ]
+    `);
+  });
 });