SQLite implementation of query support for DB based index

packages/host/app/lib/SQLiteAdapter.ts

@@ -14,6 +14,10 @@ export default class SQLiteAdapter implements DBAdapter {
   private _sqlite: typeof SQLiteWorker | undefined;
   private _dbId: string | undefined;
 
+  // TODO: one difference that I'm seeing is that it looks like "json_each" is
+  // actually similar to "json_each_text" in postgres. i think we might need to
+  // transform the SQL we run to deal with this difference.-
+
   constructor(private schemaSQL?: string) {}
 
   async startClient() {

packages/host/tests/helpers/const.ts

@@ -0,0 +1,7 @@
+import { type RealmInfo } from '@cardstack/runtime-common';
+export const testRealmURL = `http://test-realm/test/`;
+export const testRealmInfo: RealmInfo = {
+  name: 'Unnamed Workspace',
+  backgroundURL: null,
+  iconURL: null,
+};

packages/host/tests/helpers/index.gts

@@ -15,9 +15,9 @@ import {
   LooseSingleCardDocument,
   baseRealm,
   createResponse,
-  RealmInfo,
   RealmPermissions,
   Deferred,
+  type RealmInfo,
   type TokenClaims,
 } from '@cardstack/runtime-common';
 
@@ -46,14 +46,14 @@ import {
   type FieldDef,
 } from 'https://cardstack.com/base/card-api';
 
+import { testRealmInfo, testRealmURL } from './const';
 import percySnapshot from './percy-snapshot';
 
 import { renderComponent } from './render-component';
 import { WebMessageStream, messageCloseHandler } from './stream';
 import visitOperatorMode from './visit-operator-mode';
 
-export { percySnapshot };
-export { visitOperatorMode };
+export { visitOperatorMode, testRealmURL, testRealmInfo, percySnapshot };
 export * from './indexer';
 
 const waiter = buildWaiter('@cardstack/host/test/helpers/index:onFetch-waiter');
@@ -144,13 +144,6 @@ export interface Dir {
   [name: string]: string | Dir;
 }
 
-export const testRealmURL = `http://test-realm/test/`;
-export const testRealmInfo: RealmInfo = {
-  name: 'Unnamed Workspace',
-  backgroundURL: null,
-  iconURL: null,
-};
-
 export interface CardDocFiles {
   [filename: string]: LooseSingleCardDocument;
 }

packages/host/tests/helpers/indexer.ts

@@ -1,27 +1,151 @@
+import isEqual from 'lodash/isEqual';
+
 import {
   IndexerDBClient,
   asExpressions,
   addExplicitParens,
   separatedByCommas,
+  loaderFor,
+  internalKeyFor,
+  Deferred,
+  identifyCard,
+  apiFor,
+  loadCard,
+  baseCardRef,
+  type CodeRef,
+  type CardResource,
+  type Expression,
   type IndexedCardsTable,
   type RealmVersionsTable,
 } from '@cardstack/runtime-common';
 
+import type { CardDef } from 'https://cardstack.com/base/card-api';
+
+import { testRealmURL } from './const';
+
+let defaultIndexEntry = {
+  realm_version: 1,
+  realm_url: testRealmURL,
+};
+
+let typesCache = new WeakMap<typeof CardDef, Promise<string[]>>();
+
+// this leverages the logic from current-run.ts to generate the types for a card
+// that are serialized in the same manner as they appear in the index
+export async function getTypes(instance: CardDef): Promise<string[]> {
+  let loader = loaderFor(instance);
+  let card = Reflect.getPrototypeOf(instance)!.constructor as typeof CardDef;
+  let cached = typesCache.get(card);
+  if (cached) {
+    return await cached;
+  }
+  let ref = identifyCard(card);
+  if (!ref) {
+    throw new Error(`could not identify card ${card.name}`);
+  }
+  let deferred = new Deferred<string[]>();
+  typesCache.set(card, deferred.promise);
+  let types: string[] = [];
+  let fullRef: CodeRef = ref;
+  while (fullRef) {
+    let loadedCard, loadedCardRef;
+    loadedCard = await loadCard(fullRef, { loader });
+    loadedCardRef = identifyCard(loadedCard);
+    if (!loadedCardRef) {
+      throw new Error(`could not identify card ${loadedCard.name}`);
+    }
+    types.push(internalKeyFor(loadedCardRef, undefined));
+    if (!isEqual(loadedCardRef, baseCardRef)) {
+      fullRef = {
+        type: 'ancestorOf',
+        card: loadedCardRef,
+      };
+    } else {
+      break;
+    }
+  }
+  deferred.fulfill(types);
+  return types;
+}
+
+export async function serializeCard(card: CardDef): Promise<CardResource> {
+  let api = await apiFor(card);
+  return api.serializeCard(card).data as CardResource;
+}
+
+type TestIndexRow =
+  | (Pick<IndexedCardsTable, 'card_url'> &
+      Partial<Omit<IndexedCardsTable, 'card_url'>>)
+  | CardDef
+  | {
+      card: CardDef;
+      data: Partial<
+        Omit<IndexedCardsTable, 'card_url' | 'pristine_doc' | 'types'>
+      >;
+    };
+
+// There are 3 ways to setup an index:
+// 1. provide the raw data for each row in the indexed_cards table
+// 2. provide a card instance for each row in the indexed_cards table
+// 3. provide an object { card, data } where the card instance is used for each
+//    row in the indexed_cards table, as well as any additional fields that you
+//    wish to set from the `data` object.
+//
+// the realm version table will default to version 1 of the testRealmURL if no
+// value is supplied
+export async function setupIndex(
+  client: IndexerDBClient,
+  indexRows: TestIndexRow[],
+): Promise<void>;
 export async function setupIndex(
   client: IndexerDBClient,
   versionRows: RealmVersionsTable[],
-  // only assert that the non-null columns need to be present in rows objects
-  indexRows: (Pick<
-    IndexedCardsTable,
-    'card_url' | 'realm_version' | 'realm_url'
-  > &
-    Partial<
-      Omit<IndexedCardsTable, 'card_url' | 'realm_version' | 'realm_url'>
-    >)[],
-) {
-  let indexedCardsExpressions = indexRows.map((r) =>
-    asExpressions(r, {
-      jsonFields: ['deps', 'types', 'pristine_doc', 'error_doc', 'search_doc'],
+  indexRows: TestIndexRow[],
+): Promise<void>;
+export async function setupIndex(
+  client: IndexerDBClient,
+  maybeVersionRows: RealmVersionsTable[] | TestIndexRow[],
+  indexRows?: TestIndexRow[],
+): Promise<void> {
+  let versionRows: RealmVersionsTable[];
+  if (!indexRows) {
+    versionRows = [{ realm_url: testRealmURL, current_version: 1 }];
+    indexRows = maybeVersionRows as TestIndexRow[];
+  } else {
+    versionRows = maybeVersionRows as RealmVersionsTable[];
+  }
+  let indexedCardsExpressions = await Promise.all(
+    indexRows.map(async (r) => {
+      let row: Pick<IndexedCardsTable, 'card_url'> &
+        Partial<Omit<IndexedCardsTable, 'card_url'>>;
+      if ('card_url' in r) {
+        row = r;
+      } else if ('card' in r) {
+        row = {
+          card_url: r.card.id,
+          pristine_doc: await serializeCard(r.card),
+          types: await getTypes(r.card),
+          ...r.data,
+        };
+      } else {
+        row = {
+          card_url: r.id,
+          pristine_doc: await serializeCard(r),
+          types: await getTypes(r),
+        };
+      }
+      return asExpressions(
+        { ...defaultIndexEntry, ...row },
+        {
+          jsonFields: [
+            'deps',
+            'types',
+            'pristine_doc',
+            'error_doc',
+            'search_doc',
+          ],
+        },
+      );
     }),
   );
   let versionExpressions = versionRows.map((r) => asExpressions(r));
@@ -38,7 +162,7 @@ export async function setupIndex(
           addExplicitParens(separatedByCommas(row.valueExpressions)),
         ),
       ),
-    ]);
+    ] as Expression);
   }
 
   if (versionExpressions.length > 0) {
@@ -53,6 +177,6 @@ export async function setupIndex(
           addExplicitParens(separatedByCommas(row.valueExpressions)),
         ),
       ),
-    ]);
+    ] as Expression);
   }
 }

packages/host/tests/unit/index-db-test.ts → packages/host/tests/unit/indexer-test.ts

@@ -17,7 +17,7 @@ const testRealmURL2 = `http://test-realm/test2/`;
 
 let { sqlSchema } = ENV;
 
-module('Unit | index-db', function (hooks) {
+module('Unit | indexer', function (hooks) {
   let adapter: SQLiteAdapter;
   let client: IndexerDBClient;
 

packages/host/tests/unit/query-test.ts

@@ -0,0 +1,142 @@
+import { module, test, skip } from 'qunit';
+
+import {
+  Loader,
+  VirtualNetwork,
+  baseRealm,
+  IndexerDBClient,
+} from '@cardstack/runtime-common';
+
+import ENV from '@cardstack/host/config/environment';
+import SQLiteAdapter from '@cardstack/host/lib/SQLiteAdapter';
+import { shimExternals } from '@cardstack/host/lib/externals';
+
+import { CardDef } from 'https://cardstack.com/base/card-api';
+
+import { testRealmURL, setupIndex, serializeCard } from '../helpers';
+
+let cardApi: typeof import('https://cardstack.com/base/card-api');
+let string: typeof import('https://cardstack.com/base/string');
+let { sqlSchema, resolvedBaseRealmURL } = ENV;
+
+module('Unit | query', function (hooks) {
+  let adapter: SQLiteAdapter;
+  let client: IndexerDBClient;
+  let loader: Loader;
+  let testCards: { [name: string]: CardDef } = {};
+
+  hooks.beforeEach(async function () {
+    let virtualNetwork = new VirtualNetwork();
+    loader = virtualNetwork.createLoader();
+    loader.addURLMapping(new URL(baseRealm.url), new URL(resolvedBaseRealmURL));
+    shimExternals(virtualNetwork);
+
+    cardApi = await loader.import(`${baseRealm.url}card-api`);
+    string = await loader.import(`${baseRealm.url}string`);
+
+    let { field, contains, CardDef } = cardApi;
+    let { default: StringField } = string;
+    class Person extends CardDef {
+      @field name = contains(StringField);
+    }
+    class FancyPerson extends Person {
+      @field favoriteColor = contains(StringField);
+    }
+    class Cat extends CardDef {
+      @field name = contains(StringField);
+    }
+
+    loader.shimModule(`${testRealmURL}person`, { Person });
+    loader.shimModule(`${testRealmURL}fancy-person`, { FancyPerson });
+    loader.shimModule(`${testRealmURL}cat`, { Cat });
+
+    let mango = new FancyPerson({ id: `${testRealmURL}mango`, name: 'Mango' });
+    let vangogh = new Person({
+      id: `${testRealmURL}vangogh`,
+      name: 'Van Gogh',
+    });
+    let paper = new Cat({ id: `${testRealmURL}paper`, name: 'Paper' });
+    testCards = {
+      mango,
+      vangogh,
+      paper,
+    };
+
+    adapter = new SQLiteAdapter(sqlSchema);
+    client = new IndexerDBClient(adapter);
+    await client.ready();
+  });
+
+  hooks.afterEach(async function () {
+    await client.teardown();
+  });
+
+  test('can get all cards with empty filter', async function (assert) {
+    let { mango, vangogh, paper } = testCards;
+    await setupIndex(client, [mango, vangogh, paper]);
+
+    let { cards, meta } = await client.search({}, loader);
+    assert.strictEqual(meta.page.total, 3, 'the total results meta is correct');
+    assert.deepEqual(
+      cards,
+      [
+        await serializeCard(mango),
+        await serializeCard(paper),
+        await serializeCard(vangogh),
+      ],
+      'results are correct',
+    );
+  });
+
+  test('can filter by type', async function (assert) {
+    let { mango, vangogh, paper } = testCards;
+    await setupIndex(client, [mango, vangogh, paper]);
+
+    let { cards, meta } = await client.search(
+      {
+        filter: {
+          type: { module: `${testRealmURL}person`, name: 'Person' },
+        },
+      },
+      loader,
+    );
+
+    assert.strictEqual(meta.page.total, 2, 'the total results meta is correct');
+    assert.deepEqual(
+      cards,
+      [await serializeCard(mango), await serializeCard(vangogh)],
+      'results are correct',
+    );
+  });
+
+  test(`can filter using 'eq'`, async function (assert) {
+    let { mango, vangogh, paper } = testCards;
+    await setupIndex(client, [
+      { card: mango, data: { search_doc: { name: 'Mango' } } },
+      { card: vangogh, data: { search_doc: { name: 'Van Gogh' } } },
+      // this card's "name" field doesn't match our filter since our filter
+      // specified "name" fields of Person cards
+      { card: paper, data: { search_doc: { name: 'Mango' } } },
+    ]);
+
+    let { cards, meta } = await client.search(
+      {
+        filter: {
+          eq: { name: 'Mango' },
+          on: { module: `${testRealmURL}person`, name: 'Person' },
+        },
+      },
+      loader,
+    );
+
+    assert.strictEqual(meta.page.total, 1, 'the total results meta is correct');
+    assert.deepEqual(
+      cards,
+      [await serializeCard(mango)],
+      'results are correct',
+    );
+  });
+
+  skip(`can filter using 'eq' thru nested fields`);
+  skip(`can leverage queryableValue hook in card definition`);
+});