SQLite implementation of query support for DB based index

packages/base/code-ref.gts

@@ -12,6 +12,7 @@ import {
   type SerializeOpts,
   type JSONAPISingleResourceDocument,
 } from './card-api';
+import { ResolvedCodeRef } from '@cardstack/runtime-common';
 
 class BaseView extends Component<typeof CodeRefField> {
   <template>
@@ -24,38 +25,39 @@ class BaseView extends Component<typeof CodeRefField> {
   </template>
 }
 
-type CardId = { name: string; module: string };
-
 export default class CodeRefField extends FieldDef {
-  static [primitive]: CardId;
+  static [primitive]: ResolvedCodeRef;
 
   static [serialize](
-    cardRef: CardId,
+    codeRef: ResolvedCodeRef,
     _doc: JSONAPISingleResourceDocument,
     _visited?: Set<string>,
     opts?: SerializeOpts,
   ) {
     return {
-      ...cardRef,
+      ...codeRef,
       ...(opts?.maybeRelativeURL
-        ? { module: opts.maybeRelativeURL(cardRef.module) }
+        ? { module: opts.maybeRelativeURL(codeRef.module) }
         : {}),
     };
   }
   static async [deserialize]<T extends BaseDefConstructor>(
     this: T,
-    cardRef: CardId,
+    codeRef: ResolvedCodeRef,
   ): Promise<BaseInstanceType<T>> {
-    return { ...cardRef } as BaseInstanceType<T>; // return a new object so that the model cannot be mutated from the outside
+    return { ...codeRef } as BaseInstanceType<T>; // return a new object so that the model cannot be mutated from the outside
   }
-  static [queryableValue](cardRef: CardId | undefined, stack: CardDef[] = []) {
-    if (cardRef) {
+  static [queryableValue](
+    codeRef: ResolvedCodeRef | undefined,
+    stack: CardDef[] = [],
+  ) {
+    if (codeRef) {
       // if a stack is passed in, use the containing card to resolve relative references
       let moduleHref =
         stack.length > 0
-          ? new URL(cardRef.module, stack[0][relativeTo]).href
-          : cardRef.module;
-      return `${moduleHref}/${cardRef.name}`;
+          ? new URL(codeRef.module, stack[0][relativeTo]).href
+          : codeRef.module;
+      return `${moduleHref}/${codeRef.name}`;
     }
     return undefined;
   }

packages/host/app/lib/SQLiteAdapter.ts

@@ -14,6 +14,10 @@ export default class SQLiteAdapter implements DBAdapter {
   private _sqlite: typeof SQLiteWorker | undefined;
   private _dbId: string | undefined;
 
+  // TODO: one difference that I'm seeing is that it looks like "json_each" is
+  // actually similar to "json_each_text" in postgres. i think we might need to
+  // transform the SQL we run to deal with this difference.-
+
   constructor(private schemaSQL?: string) {}
 
   async startClient() {

packages/host/tests/helpers/const.ts

@@ -0,0 +1,7 @@
+import { type RealmInfo } from '@cardstack/runtime-common';
+export const testRealmURL = `http://test-realm/test/`;
+export const testRealmInfo: RealmInfo = {
+  name: 'Unnamed Workspace',
+  backgroundURL: null,
+  iconURL: null,
+};

packages/host/tests/helpers/index.gts

@@ -15,9 +15,9 @@ import {
   LooseSingleCardDocument,
   baseRealm,
   createResponse,
-  RealmInfo,
   RealmPermissions,
   Deferred,
+  type RealmInfo,
   type TokenClaims,
 } from '@cardstack/runtime-common';
 
@@ -46,14 +46,14 @@ import {
   type FieldDef,
 } from 'https://cardstack.com/base/card-api';
 
+import { testRealmInfo, testRealmURL } from './const';
 import percySnapshot from './percy-snapshot';
 
 import { renderComponent } from './render-component';
 import { WebMessageStream, messageCloseHandler } from './stream';
 import visitOperatorMode from './visit-operator-mode';
 
-export { percySnapshot };
-export { visitOperatorMode };
+export { visitOperatorMode, testRealmURL, testRealmInfo, percySnapshot };
 export * from './indexer';
 
 const waiter = buildWaiter('@cardstack/host/test/helpers/index:onFetch-waiter');
@@ -144,13 +144,6 @@ export interface Dir {
   [name: string]: string | Dir;
 }
 
-export const testRealmURL = `http://test-realm/test/`;
-export const testRealmInfo: RealmInfo = {
-  name: 'Unnamed Workspace',
-  backgroundURL: null,
-  iconURL: null,
-};
-
 export interface CardDocFiles {
   [filename: string]: LooseSingleCardDocument;
 }

packages/host/tests/helpers/indexer.ts

@@ -1,27 +1,151 @@
+import isEqual from 'lodash/isEqual';
+
 import {
   IndexerDBClient,
   asExpressions,
   addExplicitParens,
   separatedByCommas,
+  loaderFor,
+  internalKeyFor,
+  Deferred,
+  identifyCard,
+  apiFor,
+  loadCard,
+  baseCardRef,
+  type CodeRef,
+  type CardResource,
+  type Expression,
   type IndexedCardsTable,
   type RealmVersionsTable,
 } from '@cardstack/runtime-common';
 
+import type { CardDef } from 'https://cardstack.com/base/card-api';
+
+import { testRealmURL } from './const';
+
+let defaultIndexEntry = {
+  realm_version: 1,
+  realm_url: testRealmURL,
+};
+
+let typesCache = new WeakMap<typeof CardDef, Promise<string[]>>();
+
+// this leverages the logic from current-run.ts to generate the types for a card
+// that are serialized in the same manner as they appear in the index
+export async function getTypes(instance: CardDef): Promise<string[]> {
+  let loader = loaderFor(instance);
+  let card = Reflect.getPrototypeOf(instance)!.constructor as typeof CardDef;
+  let cached = typesCache.get(card);
+  if (cached) {
+    return await cached;
+  }
+  let ref = identifyCard(card);
+  if (!ref) {
+    throw new Error(`could not identify card ${card.name}`);
+  }
+  let deferred = new Deferred<string[]>();
+  typesCache.set(card, deferred.promise);
+  let types: string[] = [];
+  let fullRef: CodeRef = ref;
+  while (fullRef) {
+    let loadedCard, loadedCardRef;
+    loadedCard = await loadCard(fullRef, { loader });
+    loadedCardRef = identifyCard(loadedCard);
+    if (!loadedCardRef) {
+      throw new Error(`could not identify card ${loadedCard.name}`);
+    }
+    types.push(internalKeyFor(loadedCardRef, undefined));
+    if (!isEqual(loadedCardRef, baseCardRef)) {
+      fullRef = {
+        type: 'ancestorOf',
+        card: loadedCardRef,
+      };
+    } else {
+      break;
+    }
+  }
+  deferred.fulfill(types);
+  return types;
+}
+
+export async function serializeCard(card: CardDef): Promise<CardResource> {
+  let api = await apiFor(card);
+  return api.serializeCard(card).data as CardResource;
+}
+
+type TestIndexRow =
+  | (Pick<IndexedCardsTable, 'card_url'> &
+      Partial<Omit<IndexedCardsTable, 'card_url'>>)
+  | CardDef
+  | {
+      card: CardDef;
+      data: Partial<
+        Omit<IndexedCardsTable, 'card_url' | 'pristine_doc' | 'types'>
+      >;
+    };
+
+// There are 3 ways to setup an index:
+// 1. provide the raw data for each row in the indexed_cards table
+// 2. provide a card instance for each row in the indexed_cards table
+// 3. provide an object { card, data } where the card instance is used for each
+//    row in the indexed_cards table, as well as any additional fields that you
+//    wish to set from the `data` object.
+//
+// the realm version table will default to version 1 of the testRealmURL if no
+// value is supplied
+export async function setupIndex(
+  client: IndexerDBClient,
+  indexRows: TestIndexRow[],
+): Promise<void>;
 export async function setupIndex(
   client: IndexerDBClient,
   versionRows: RealmVersionsTable[],
-  // only assert that the non-null columns need to be present in rows objects
-  indexRows: (Pick<
-    IndexedCardsTable,
-    'card_url' | 'realm_version' | 'realm_url'
-  > &
-    Partial<
-      Omit<IndexedCardsTable, 'card_url' | 'realm_version' | 'realm_url'>
-    >)[],
-) {
-  let indexedCardsExpressions = indexRows.map((r) =>
-    asExpressions(r, {
-      jsonFields: ['deps', 'types', 'pristine_doc', 'error_doc', 'search_doc'],
+  indexRows: TestIndexRow[],
+): Promise<void>;
+export async function setupIndex(
+  client: IndexerDBClient,
+  maybeVersionRows: RealmVersionsTable[] | TestIndexRow[],
+  indexRows?: TestIndexRow[],
+): Promise<void> {
+  let versionRows: RealmVersionsTable[];
+  if (!indexRows) {
+    versionRows = [{ realm_url: testRealmURL, current_version: 1 }];
+    indexRows = maybeVersionRows as TestIndexRow[];
+  } else {
+    versionRows = maybeVersionRows as RealmVersionsTable[];
+  }
+  let indexedCardsExpressions = await Promise.all(
+    indexRows.map(async (r) => {
+      let row: Pick<IndexedCardsTable, 'card_url'> &
+        Partial<Omit<IndexedCardsTable, 'card_url'>>;
+      if ('card_url' in r) {
+        row = r;
+      } else if ('card' in r) {
+        row = {
+          card_url: r.card.id,
+          pristine_doc: await serializeCard(r.card),
+          types: await getTypes(r.card),
+          ...r.data,
+        };
+      } else {
+        row = {
+          card_url: r.id,
+          pristine_doc: await serializeCard(r),
+          types: await getTypes(r),
+        };
+      }
+      return asExpressions(
+        { ...defaultIndexEntry, ...row },
+        {
+          jsonFields: [
+            'deps',
+            'types',
+            'pristine_doc',
+            'error_doc',
+            'search_doc',
+          ],
+        },
+      );
     }),
   );
   let versionExpressions = versionRows.map((r) => asExpressions(r));
@@ -38,7 +162,7 @@ export async function setupIndex(
           addExplicitParens(separatedByCommas(row.valueExpressions)),
         ),
       ),
-    ]);
+    ] as Expression);
   }
 
   if (versionExpressions.length > 0) {
@@ -53,6 +177,6 @@ export async function setupIndex(
           addExplicitParens(separatedByCommas(row.valueExpressions)),
         ),
       ),
-    ]);
+    ] as Expression);
   }
 }

packages/host/tests/unit/index-db-test.ts → packages/host/tests/unit/indexer-test.ts

@@ -17,7 +17,7 @@ const testRealmURL2 = `http://test-realm/test2/`;
 
 let { sqlSchema } = ENV;
 
-module('Unit | index-db', function (hooks) {
+module('Unit | indexer', function (hooks) {
   let adapter: SQLiteAdapter;
   let client: IndexerDBClient;