initial commit

.github/workflows/nodejs.yml

@@ -0,0 +1,62 @@
+name: Node build, test and publish
+
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup stale actions
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js 18
+      uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: yarn
+    - name: npm install, lint, build, and test
+      run: |
+        yarn install --immutable
+        yarn lint
+        yarn build
+        yarn test
+      env:
+        CI: true
+
+  publish-npm:
+    needs: build
+    if: github.ref == 'refs/heads/main'
+    permissions:
+      contents: write
+      issues: write
+      id-token: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: yarn
+      - run: yarn install --immutable
+      - run: yarn build
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.SESAMECARE_OSS_NPM_TOKEN }}
+        run: |
+          yarn dlx semantic-release

.gitignore

@@ -0,0 +1,68 @@
+# Custom
+src/generated
+.transpiled
+.nyc_output
+.eslintcache
+.vscode
+
+# TypeScript incremental compilation cache
+*.tsbuildinfo
+
+# Stock
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+*.orig
+
+work
+/build
+pids
+logs
+results
+coverage
+lib-cov
+html-report
+xunit.xml
+node_modules
+npm-debug.log
+
+.project
+.idea
+.settings
+.iml
+*.sublime-workspace
+*.sublime-project
+
+.DS_Store*
+ehthumbs.db
+Icon?
+Thumbs.db
+.AppleDouble
+.LSOverride
+.Spotlight-V100
+.Trashes
+
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+.node_repl_history
+
+# TypeScript incremental compilation cache
+*.tsbuildinfo
+# Added by coconfig
+.eslintignore
+.npmignore
+tsconfig.json
+tsconfig.build.json
+.prettierrc.js
+.eslintrc.js
+.commitlintrc.json
+vitest.config.ts

.yarnrc.yml

@@ -0,0 +1,7 @@
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+    spec: "@yarnpkg/plugin-interactive-tools"
+
+yarnPath: .yarn/releases/yarn-3.8.1.cjs

README.md

@@ -0,0 +1,5 @@
+# multi-source-pager
+
+Sometimes you need to join multiple remote data sources (such as external APIs) together into one list and support paging. multi-source-pager provides a class that manages these multiple streams and exposes a cursor-based paging interface to clients.
+
+Currently, the module only supports "forward" paging, additional work is required to go backwards.

__tests__/LetterDataSource.ts

@@ -0,0 +1,41 @@
+import { DataSource } from '../src/types';
+
+export interface LetterResult {
+  cursor: string;
+  type: 'letter';
+  data: string;
+}
+
+export class MockLetterSource implements DataSource<LetterResult> {
+  constructor(private results: LetterResult[]) {}
+
+  async getResults(cursor: string | undefined, limit: number) {
+    const startIndex = this.results.findIndex(result => result.cursor > (cursor ?? ''));
+    return {
+      total: this.results.length,
+      results: startIndex === -1 ? [] : this.results.slice(startIndex, startIndex + limit),
+    };
+  }
+
+  sortKey(result: LetterResult): string {
+    return result.cursor;
+  }
+}
+
+export const mockLetters: LetterResult[] = [
+  { cursor: '2023-01-01T00:00:00.000Z#A', data: 'A', type: 'letter' },
+  { cursor: '2023-01-02T00:00:00.000Z#B', data: 'B', type: 'letter' },
+  { cursor: '2023-01-03T00:00:00.000Z#C', data: 'C', type: 'letter' },
+  { cursor: '2023-01-04T00:00:00.000Z#D', data: 'D', type: 'letter' },
+  { cursor: '2023-01-05T00:00:00.000Z#E', data: 'E', type: 'letter' },
+];
+
+export const mockDoubleLetters: LetterResult[] = [
+  { cursor: '2023-01-01T00:00:00.000Z#AA', data: 'AA', type: 'letter' },
+  { cursor: '2023-01-01T00:00:00.000Z#AAA', data: 'AAA', type: 'letter' },
+  { cursor: '2023-01-02T00:00:00.000Z#BB', data: 'BB', type: 'letter' },
+  { cursor: '2023-01-02T00:00:00.000Z#BBB', data: 'BBB', type: 'letter' },
+  { cursor: '2023-01-06T00:00:00.000Z#FF', data: 'FF', type: 'letter' },
+  { cursor: '2023-01-07T00:00:00.000Z#GG', data: 'GG', type: 'letter' },
+  { cursor: '2023-01-08T00:00:00.000Z#HH', data: 'HH', type: 'letter' },
+];

package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@sesamecare-oss/multi-source-pager",
+  "version": "0.0.0",
+  "description": "A package to assemble multiple cursor-based paginated sources into a single paginated source.",
+  "main": "build/index.js",
+  "types": "build/index.d.ts",
+  "author": "Developers <developers@sesamecare.com>",
+  "license": "UNLICENSED",
+  "packageManager": "yarn@3.8.1",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "clean": "yarn dlx rimraf ./dist",
+    "lint": "eslint .",
+    "postinstall": "coconfig",
+    "test": "vitest"
+  },
+  "keywords": [
+    "typescript",
+    "sesame"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sesamecare/multi-source-pager.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "release": {
+    "branches": [
+      "main"
+    ],
+    "plugins": [
+      "@semantic-release/commit-analyzer",
+      "@semantic-release/release-notes-generator",
+      [
+        "@semantic-release/exec",
+        {
+          "publishCmd": "yarn dlx pinst --disable"
+        }
+      ],
+      "@semantic-release/npm",
+      "@semantic-release/github"
+    ]
+  },
+  "config": {
+    "coconfig": "@openapi-typescript-infra/coconfig"
+  },
+  "devDependencies": {
+    "@openapi-typescript-infra/coconfig": "^4.3.0",
+    "@semantic-release/exec": "^6.0.3",
+    "@semantic-release/github": "^10.0.2",
+    "@types/node": "^20.11.29",
+    "@typescript-eslint/eslint-plugin": "^7.3.1",
+    "@typescript-eslint/parser": "^7.3.1",
+    "coconfig": "^1.4.1",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-import-resolver-typescript": "^3.6.1",
+    "eslint-plugin-import": "^2.29.1",
+    "typescript": "^5.4.2",
+    "vitest": "^1.4.0"
+  },
+  "dependencies": {
+    "fflate": "^0.8.2",
+    "universal-base64": "^2.1.0"
+  }
+}

src/index.spec.ts

@@ -0,0 +1,121 @@
+import { describe, it, expect } from 'vitest';
+
+import { MockLetterSource, mockDoubleLetters, mockLetters } from '../__tests__/LetterDataSource';
+
+import { multiSourcePager } from './index';
+
+// Comparator for sorting by cursor (ISO date)
+const comparator = (a: string, b: string) => a.localeCompare(b);
+
+describe('multiSourcePager', () => {
+  it('should return the first page of results sorted by date', async () => {
+    const dataSourceA = new MockLetterSource(mockLetters);
+    const dataSourceB = new MockLetterSource(mockDoubleLetters);
+
+    const p1 = await multiSourcePager({
+      comparator,
+      pageSize: 3,
+    }, dataSourceA, dataSourceB);
+
+    expect(p1.results.length).toBe(3);
+    expect(p1.total).toBe(12);
+    expect(p1.results).toMatchInlineSnapshot(`
+      [
+        {
+          "cursor": "2023-01-01T00:00:00.000Z#A",
+          "data": "A",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-01T00:00:00.000Z#AA",
+          "data": "AA",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-01T00:00:00.000Z#AAA",
+          "data": "AAA",
+          "type": "letter",
+        },
+      ]
+    `);
+    expect(p1.cursor).toBe('WyIyMDIzLTAxLTAxVDAwOjAwOjAwLjAwMFojQSIsIjIwMjMtMDEtMDFUMDA6MDA6MDAuMDAwWiNBQUEiXQ==');
+
+    const p2 = await multiSourcePager({
+      comparator,
+      pageSize: 3,
+      cursor: p1.cursor,
+    }, dataSourceA, dataSourceB);
+
+    expect(p2.results.length).toBe(3);
+    expect(p2.total).toBe(12);
+    expect(p2.results).toMatchInlineSnapshot(`
+      [
+        {
+          "cursor": "2023-01-02T00:00:00.000Z#B",
+          "data": "B",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-02T00:00:00.000Z#BB",
+          "data": "BB",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-02T00:00:00.000Z#BBB",
+          "data": "BBB",
+          "type": "letter",
+        },
+      ]
+    `);
+
+    const p3 = await multiSourcePager({
+      comparator,
+      pageSize: 5,
+      cursor: p2.cursor,
+    }, dataSourceA, dataSourceB);
+    expect(p3.results).toMatchInlineSnapshot(`
+      [
+        {
+          "cursor": "2023-01-03T00:00:00.000Z#C",
+          "data": "C",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-04T00:00:00.000Z#D",
+          "data": "D",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-05T00:00:00.000Z#E",
+          "data": "E",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-06T00:00:00.000Z#FF",
+          "data": "FF",
+          "type": "letter",
+        },
+        {
+          "cursor": "2023-01-07T00:00:00.000Z#GG",
+          "data": "GG",
+          "type": "letter",
+        },
+      ]
+    `);
+
+    const p4 = await multiSourcePager({
+      comparator,
+      pageSize: 5,
+      cursor: p3.cursor,
+    }, dataSourceA, dataSourceB);
+    expect(p4.results).toMatchInlineSnapshot(`
+      [
+        {
+          "cursor": "2023-01-08T00:00:00.000Z#HH",
+          "data": "HH",
+          "type": "letter",
+        },
+      ]
+    `);
+  });
+});

src/index.ts

@@ -0,0 +1,2 @@
+export * from './search';
+export * from './types';