Add form_documents table and an initializeForm service, responsible f…

…or creating a form that's initialized with data from a PDF. initializeForm is not currently wired to the UI.

packages/database/migrations/20241031103354_form_documents_table.mjs

@@ -0,0 +1,21 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function up(knex) {
+  await knex.schema.createTable('form_documents', table => {
+    table.uuid('id').primary();
+    table.string('type').notNullable();
+    table.string('file_name').notNullable();
+    table.binary('data').notNullable();
+    table.string('extract').notNullable();
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export async function down(knex) {
+  await knex.schema.dropTableIfExists('form_documents');
+}

packages/database/src/clients/kysely/types.ts

@@ -1,4 +1,5 @@
 import type {
+  ColumnType,
   Generated,
   Insertable,
   Kysely,
@@ -14,7 +15,9 @@ export interface Database<T extends Engine = Engine> {
   sessions: SessionsTable<T>;
   forms: FormsTable;
   form_sessions: FormSessionsTable;
+  form_documents: FormDocumentsTable;
 }
+export type DatabaseClient = Kysely<Database>;
 
 interface UsersTable {
   id: string;
@@ -48,8 +51,6 @@ export type FormsTableSelectable = Selectable<FormsTable>;
 export type FormsTableInsertable = Insertable<FormsTable>;
 export type FormsTableUpdateable = Updateable<FormsTable>;
 
-export type DatabaseClient = Kysely<Database>;
-
 interface FormSessionsTable {
   id: string;
   form_id: string;
@@ -60,3 +61,14 @@ interface FormSessionsTable {
 export type FormSessionsTableSelectable = Selectable<FormSessionsTable>;
 export type FormSessionsTableInsertable = Insertable<FormSessionsTable>;
 export type FormSessionsTableUpdateable = Updateable<FormSessionsTable>;
+
+interface FormDocumentsTable {
+  id: string;
+  type: string;
+  data: ColumnType<Buffer, Buffer, Buffer>;
+  file_name: string;
+  extract: string;
+}
+export type FormDocumentsTableSelectable = Selectable<FormDocumentsTable>;
+export type FormDocumentsTableInsertable = Insertable<FormDocumentsTable>;
+export type FormDocumentsTableUpdateable = Updateable<FormDocumentsTable>;

packages/forms/src/builder/index.ts

@@ -9,7 +9,7 @@ import {
   removePatternFromBlueprint,
   updateFormSummary,
 } from '../blueprint.js';
-import { addDocument } from '../documents/document.js';
+import { addDocument, addParsedPdfToForm } from '../documents/document.js';
 import type { FormErrors } from '../error.js';
 import {
   createDefaultPattern,
@@ -23,6 +23,8 @@ import {
 import { type FieldsetPattern } from '../patterns/fieldset/config.js';
 import { type PageSetPattern } from '../patterns/page-set/config.js';
 import type { Blueprint, FormSummary } from '../types.js';
+import type { ParsedPdf } from '../documents/pdf/parsing-api.js';
+import type { DocumentFieldMap } from '../documents/types.js';
 
 export class BlueprintBuilder {
   bp: Blueprint;
@@ -47,6 +49,15 @@ export class BlueprintBuilder {
     this.bp = updatedForm;
   }
 
+  async addDocumentRef(opts: { id: string; extract: ParsedPdf }) {
+    const { updatedForm } = await addParsedPdfToForm(this.form, {
+      id: opts.id,
+      label: opts.extract.title,
+      extract: opts.extract,
+    });
+    this.bp = updatedForm;
+  }
+
   addPage() {
     const newPage = createDefaultPattern(this.config, 'page');
     this.bp = addPageToPageSet(this.form, newPage);

packages/forms/src/context/index.ts

@@ -1,10 +1,13 @@
-import { type FormConfig } from '../pattern.js';
-import { type FormRepository } from '../repository/index.js';
+import type { ParsePdf } from '../documents/index.js';
+import type { FormConfig } from '../pattern.js';
+import type { FormRepository } from '../repository/index.js';
 
-export { createTestBrowserFormService } from './test/index.js';
 export { BrowserFormRepository } from './browser/form-repo.js';
+export { createTestBrowserFormService } from './test/index.js';
+
 export type FormServiceContext = {
   repository: FormRepository;
   config: FormConfig;
   isUserLoggedIn: () => boolean;
+  parsePdf: ParsePdf;
 };

packages/forms/src/documents/__tests__/doj-pardon-marijuana.test.ts

@@ -3,7 +3,8 @@ import { describe, expect, test } from 'vitest';
 import { Success } from '@atj/common';
 
 import { type DocumentFieldMap } from '../index.js';
-import { fillPDF, getDocumentFieldData } from '../pdf/index.js';
+import { fillPDF } from '../pdf/index.js';
+import { getDocumentFieldData } from '../pdf/extract.js';
 
 import { loadSamplePDF } from './sample-data.js';
 

packages/forms/src/documents/__tests__/extract.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
-import { getDocumentFieldData } from '../index.js';
 import { loadSamplePDF } from './sample-data.js';
+import { getDocumentFieldData } from '../pdf/extract.js';
 
 describe('PDF form field extraction', () => {
   it('extracts data from California UD-105 form', async () => {

packages/forms/src/documents/__tests__/fill-pdf.test.ts

@@ -2,8 +2,9 @@ import { beforeAll, describe, expect, it } from 'vitest';
 
 import { type Failure, type Success } from '@atj/common';
 
-import { getDocumentFieldData, fillPDF } from '../index.js';
+import { fillPDF } from '../index.js';
 import { loadSamplePDF } from './sample-data.js';
+import { getDocumentFieldData } from '../pdf/extract.js';
 
 describe('PDF form filler', () => {
   let pdfBytes: Uint8Array;

packages/forms/src/documents/document.ts

@@ -8,18 +8,47 @@ import { type Pattern } from '../pattern.js';
 import { type InputPattern } from '../patterns/input/config.js';
 import { type SequencePattern } from '../patterns/sequence.js';
 import { type Blueprint } from '../types.js';
+import { getDocumentFieldData } from './pdf/extract.js';
 
-import { type PDFDocument, getDocumentFieldData } from './pdf/index.js';
+import { type PDFDocument } from './pdf/index.js';
 import {
   type FetchPdfApiResponse,
-  processApiResponse,
+  type ParsedPdf,
   fetchPdfApiResponse,
+  processApiResponse,
 } from './pdf/parsing-api.js';
 
 import { type DocumentFieldMap } from './types.js';
 
 export type DocumentTemplate = PDFDocument;
 
+export const addParsedPdfToForm = async (
+  form: Blueprint,
+  document: {
+    id: string;
+    label: string;
+    extract: ParsedPdf;
+  }
+) => {
+  form = addPatternMap(form, document.extract.patterns, document.extract.root);
+  const updatedForm = addFormOutput(form, {
+    id: document.id,
+    data: new Uint8Array(), // TODO: remove this no-longer-used field
+    path: document.label,
+    fields: document.extract.outputs,
+    formFields: Object.fromEntries(
+      Object.keys(document.extract.outputs).map(output => {
+        return [output, document.extract.outputs[output].name];
+      })
+    ),
+  });
+  return {
+    newFields: document.extract.outputs,
+    updatedForm,
+    errors: document.extract.errors,
+  };
+};
+
 export const addDocument = async (
   form: Blueprint,
   fileDetails: {
@@ -41,6 +70,7 @@ export const addDocument = async (
     });
     form = addPatternMap(form, parsedPdf.patterns, parsedPdf.root);
     const updatedForm = addFormOutput(form, {
+      id: 'document-1', // TODO: generate a unique ID
       data: fileDetails.data,
       path: fileDetails.name,
       fields: parsedPdf.outputs,
@@ -58,6 +88,7 @@ export const addDocument = async (
   } else {
     const formWithFields = addDocumentFieldsToForm(form, fields);
     const updatedForm = addFormOutput(formWithFields, {
+      id: 'document-1', // TODO: generate a unique ID
       data: fileDetails.data,
       path: fileDetails.name,
       fields,

packages/forms/src/documents/pdf/index.ts

@@ -1,4 +1,11 @@
-export { getDocumentFieldData } from './extract.js';
+import { getDocumentFieldData } from './extract.js';
+import {
+  type ParsedPdf,
+  fetchPdfApiResponse,
+  processApiResponse,
+} from './parsing-api.js';
+import type { DocumentFieldMap } from '../types.js';
+
 export * from './generate.js';
 export { generateDummyPDF } from './generate-dummy.js';
 
@@ -20,3 +27,14 @@ export type PDFFieldType =
   | 'RadioGroup'
   | 'Paragraph'
   | 'RichText';
+
+export type ParsePdf = (
+  pdf: Uint8Array
+) => Promise<{ parsedPdf: ParsedPdf; fields: DocumentFieldMap }>;
+
+export const parsePdf: ParsePdf = async (pdfBytes: Uint8Array) => {
+  const fields = await getDocumentFieldData(pdfBytes);
+  const apiResponse = await fetchPdfApiResponse(pdfBytes);
+  const parsedPdf = await processApiResponse(apiResponse);
+  return { parsedPdf, fields };
+};

packages/forms/src/documents/types.ts

@@ -1,3 +1,12 @@
+import type { Result } from '@atj/common';
+
+import { getDocumentFieldData } from './pdf/extract';
+import {
+  type ParsedPdf,
+  fetchPdfApiResponse,
+  processApiResponse,
+} from './pdf/parsing-api';
+
 export type DocumentFieldValue =
   | {
       type: 'TextField';

packages/forms/src/repository/add-document.test.ts

@@ -0,0 +1,29 @@
+import { beforeAll, expect, it, vi } from 'vitest';
+
+import { type DbTestContext, describeDatabase } from '@atj/database/testing';
+import { addDocument } from './add-document.js';
+import type { ParsedPdf } from '../documents/pdf/parsing-api.js';
+import type { DocumentFieldMap } from '../documents/types.js';
+
+describeDatabase('add document', () => {
+  const today = new Date(2000, 1, 1);
+
+  beforeAll(async () => {
+    vi.setSystemTime(today);
+  });
+
+  it<DbTestContext>('works', async ({ db }) => {
+    const result = await addDocument(db.ctx, {
+      fileName: 'file.pdf',
+      data: new Uint8Array([1, 2, 3]),
+      extract: {
+        parsedPdf: {} as ParsedPdf,
+        fields: {} as DocumentFieldMap,
+      },
+    });
+    if (result.success === false) {
+      expect.fail(`addDocument failed: ${result.error}`);
+    }
+    expect(result.data.id).toBeTypeOf('string');
+  });
+});

packages/forms/src/repository/add-document.ts

@@ -0,0 +1,39 @@
+import { type Result, failure, success } from '@atj/common';
+import { type DatabaseContext } from '@atj/database';
+
+import type { ParsedPdf } from '../documents/pdf/parsing-api';
+import type { DocumentFieldMap } from '../documents/types';
+
+export type AddDocument = (
+  ctx: DatabaseContext,
+  document: {
+    fileName: string;
+    data: Uint8Array;
+    extract: {
+      parsedPdf: ParsedPdf;
+      fields: DocumentFieldMap;
+    };
+  }
+) => Promise<Result<{ id: string }>>;
+
+export const addDocument: AddDocument = async (ctx, document) => {
+  const uuid = crypto.randomUUID();
+  const db = await ctx.getKysely();
+
+  return await db
+    .insertInto('form_documents')
+    .values({
+      id: uuid,
+      type: 'pdf',
+      file_name: document.fileName,
+      data: Buffer.from(document.data),
+      extract: JSON.stringify(document.extract),
+    })
+    .execute()
+    .then(() =>
+      success({
+        id: uuid,
+      })
+    )
+    .catch(err => failure(err.message));
+};

packages/forms/src/repository/index.ts

@@ -1,6 +1,7 @@
 import { type ServiceMethod, createService } from '@atj/common';
 import { type DatabaseContext } from '@atj/database';
 
+import { type AddDocument, addDocument } from './add-document.js';
 import { type AddForm, addForm } from './add-form.js';
 import { type DeleteForm, deleteForm } from './delete-form.js';
 import { type GetForm, getForm } from './get-form.js';
@@ -13,6 +14,7 @@ import {
 } from './upsert-form-session.js';
 
 export interface FormRepository {
+  addDocument: ServiceMethod<AddDocument>;
   addForm: ServiceMethod<AddForm>;
   deleteForm: ServiceMethod<DeleteForm>;
   getForm: ServiceMethod<GetForm>;
@@ -24,6 +26,7 @@ export interface FormRepository {
 
 export const createFormsRepository = (ctx: DatabaseContext): FormRepository =>
   createService(ctx, {
+    addDocument,
     addForm,
     deleteForm,
     getFormList,

packages/forms/src/services/index.ts

@@ -1,4 +1,4 @@
-import { createService, ServiceMethod } from '@atj/common';
+import { type ServiceMethod, createService } from '@atj/common';
 
 import { type FormServiceContext } from '../context/index.js';
 
@@ -7,6 +7,7 @@ import { type DeleteForm, deleteForm } from './delete-form.js';
 import { type GetForm, getForm } from './get-form.js';
 import { type GetFormList, getFormList } from './get-form-list.js';
 import { type GetFormSession, getFormSession } from './get-form-session.js';
+import { type InitializeForm, initializeForm } from './initialize-form.js';
 import { type SaveForm, saveForm } from './save-form.js';
 import { type SubmitForm, submitForm } from './submit-form.js';
 
@@ -17,6 +18,7 @@ export const createFormService = (ctx: FormServiceContext) =>
     getForm,
     getFormList,
     getFormSession,
+    initializeForm,
     saveForm,
     submitForm,
   });
@@ -27,6 +29,7 @@ export type FormService = {
   getForm: ServiceMethod<GetForm>;
   getFormList: ServiceMethod<GetFormList>;
   getFormSession: ServiceMethod<GetFormSession>;
+  initializeForm: ServiceMethod<InitializeForm>;
   saveForm: ServiceMethod<SaveForm>;
   submitForm: ServiceMethod<SubmitForm>;
 };