diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts index abb2d3486f..40a96f90ed 100644 --- a/firestore-bigquery-export/functions/src/index.ts +++ b/firestore-bigquery-export/functions/src/index.ts @@ -97,52 +97,79 @@ export const fsexportbigquery = functions .document(config.collectionPath) .onWrite(async (change, context) => { logs.start(); - try { - const changeType = getChangeType(change); - const documentId = getDocumentId(change); - const isCreated = changeType === ChangeType.CREATE; - const isDeleted = changeType === ChangeType.DELETE; + /** + * enqueue data cannot currently handle documentdata + * Serialize early before queueing in clopud task + * Cloud tasks currently have a limit of 1mb, this also ensures payloads are kept to a minimum + */ + const changeType = getChangeType(change); + const documentId = getDocumentId(change); - const data = isDeleted ? undefined : change.after.data(); - const oldData = - isCreated || config.excludeOldData ? undefined : change.before.data(); + const isCreated = changeType === ChangeType.CREATE; + const isDeleted = changeType === ChangeType.DELETE; - await events.recordStartEvent({ - documentId, - changeType, - before: { - data: change.before.data(), - }, - after: { - data: change.after.data(), - }, - context: context.resource, - }); + const data = isDeleted ? undefined : change.after.data(); + const oldData = + isCreated || config.excludeOldData ? undefined : change.before.data(); + + let serializedData: any; + let serializedOldData: any; + + try { + serializedData = eventTracker.serializeData(data); + } catch (err) { + functions.logger.error("failed to serialize data", err); + throw err; + } + try { + serializedOldData = eventTracker.serializeData(oldData); + } catch (err) { + functions.logger.error("failed to serialize old data", err); + throw err; + } + + try { + await events + .recordStartEvent({ + documentId, + changeType, + before: { + data: change.before.data(), + }, + after: { + data: change.after.data(), + }, + context: context.resource, + }) + .catch((err) => { + functions.logger.error("failed to record start event", err); + + throw err; + }); const queue = getFunctions().taskQueue( `locations/${config.location}/functions/syncBigQuery`, config.instanceId ); - /** - * enqueue data cannot currently handle documentdata - * Serialize early before queueing in clopud task - * Cloud tasks currently have a limit of 1mb, this also ensures payloads are kept to a minimum - */ - const seializedData = eventTracker.serializeData(data); - const serializedOldData = eventTracker.serializeData(oldData); - - await queue.enqueue({ - context, - changeType, - documentId, - data: seializedData, - oldData: serializedOldData, - }); + await queue + .enqueue({ + context, + changeType, + documentId, + data: serializedData, + oldData: serializedOldData, + }) + .catch((err) => { + functions.logger.error("failed to enqueue task", err); + + throw err; + }); } catch (err) { await events.recordErrorEvent(err as Error); - logs.error(err); + + logs.error(err, serializedData, serializedOldData); const eventAgeMs = Date.now() - Date.parse(context.timestamp); const eventMaxAgeMs = 10000; diff --git a/firestore-bigquery-export/functions/src/logs.ts b/firestore-bigquery-export/functions/src/logs.ts index ae0a931eaf..f0c616ba43 100644 --- a/firestore-bigquery-export/functions/src/logs.ts +++ b/firestore-bigquery-export/functions/src/logs.ts @@ -149,8 +149,12 @@ export const dataTypeInvalid = ( ); }; -export const error = (err: Error) => { - logger.error("Error when mirroring data to BigQuery", err); +export const error = (err: Error, data, oldData) => { + logger.error("Error when mirroring data to BigQuery", { + error: err, + data, + oldData, + }); }; export const init = () => {