From 5a9311b4c1dc793bc45be8d99ccb048262eaa371 Mon Sep 17 00:00:00 2001 From: Gustolandia Date: Fri, 20 Dec 2024 16:19:27 +0000 Subject: [PATCH] fix(firestore-bigquery-export): update event types for Eventarc compatibility (#1981) --- firestore-bigquery-export/extension.yaml | 27 +++- .../functions/__tests__/functions.test.ts | 132 ++++++++++-------- .../functions/src/events.ts | 121 +++++++++++++--- 3 files changed, 202 insertions(+), 78 deletions(-) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 8e7e3e8c8..f3dd16e87 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -408,6 +408,7 @@ params: default: 3 events: + # OLD event types for backward compatibility - type: firebase.extensions.firestore-counter.v1.onStart description: Occurs when a trigger has been called within the Extension, and will @@ -415,8 +416,8 @@ events: - type: firebase.extensions.firestore-counter.v1.onSuccess description: - Occurs when image resizing completes successfully. The event will contain - further details about specific formats and sizes. + Occurs when a task completes successfully. The event will contain further + details about specific results. - type: firebase.extensions.firestore-counter.v1.onError description: @@ -427,6 +428,28 @@ events: description: Occurs when the function is settled. Provides no customized data other than the context. + + # NEW event types following the updated naming convention + - type: firebase.extensions.firestore-bigquery-export.v1.onStart + description: + Occurs when a trigger has been called within the Extension, and will + include data such as the context of the trigger request. + + - type: firebase.extensions.firestore-bigquery-export.v1.onSuccess + description: + Occurs when a task completes successfully. The event will contain further + details about specific results. + + - type: firebase.extensions.firestore-bigquery-export.v1.onError + description: + Occurs when an issue has been experienced in the Extension. This will + include any error data that has been included within the Error Exception. + + - type: firebase.extensions.firestore-bigquery-export.v1.onCompletion + description: + Occurs when the function is settled. Provides no customized data other + than the context. + - type: firebase.extensions.big-query-export.v1.sync.start description: Occurs on a firestore document write event. diff --git a/firestore-bigquery-export/functions/__tests__/functions.test.ts b/firestore-bigquery-export/functions/__tests__/functions.test.ts index d9aefb52e..a96568767 100644 --- a/firestore-bigquery-export/functions/__tests__/functions.test.ts +++ b/firestore-bigquery-export/functions/__tests__/functions.test.ts @@ -2,17 +2,15 @@ import * as admin from "firebase-admin"; import { logger } from "firebase-functions"; import * as functionsTestInit from "../node_modules/firebase-functions-test"; import mockedEnv from "../node_modules/mocked-env"; - import { mockConsoleLog } from "./__mocks__/console"; import config from "../src/config"; +// Mock Firestore BigQuery Tracker jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({ - FirestoreBigQueryEventHistoryTracker: jest.fn(() => { - return { - record: jest.fn(() => {}), - serializeData: jest.fn(() => {}), - }; - }), + FirestoreBigQueryEventHistoryTracker: jest.fn(() => ({ + record: jest.fn(() => {}), + serializeData: jest.fn(() => {}), + })), ChangeType: { DELETE: 2, UPDATE: 1, @@ -20,55 +18,53 @@ jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({ }, })); -jest.mock("firebase-admin/functions", () => ({ - getFunctions: () => { - return { taskQueue: jest.fn() }; - }, -})); - -jest.mock("firebase-admin/functions", () => ({ - getFunctions: () => { - return { - taskQueue: jest.fn(() => { - return { enqueue: jest.fn() }; - }), - }; - }, +// Mock firebase-admin eventarc +jest.mock("firebase-admin/eventarc", () => ({ + getEventarc: jest.fn(() => ({ + channel: jest.fn(() => ({ + publish: jest.fn(), + })), + })), })); +// Mock Logs jest.mock("../src/logs", () => ({ ...jest.requireActual("../src/logs"), start: jest.fn(() => logger.log("Started execution of extension with configuration", config) ), - init: jest.fn(() => {}), - error: jest.fn(() => {}), complete: jest.fn(() => logger.log("Completed execution of extension")), })); +// Environment Variables const defaultEnvironment = { PROJECT_ID: "fake-project", DATASET_ID: "my_ds_id", TABLE_ID: "my_id", COLLECTION_PATH: "example", + EVENTARC_CHANNEL: "test-channel", // Mock Eventarc Channel + EXT_SELECTED_EVENTS: "onStart,onSuccess,onError,onCompletion", // Allowed event types }; -export const mockExport = (document, data) => { - const ref = require("../src/index").fsexportbigquery; - let functionsTest = functionsTestInit(); +let restoreEnv; +let functionsTest = functionsTestInit(); +/** Helper to Mock Export */ +const mockExport = (document, data) => { + const ref = require("../src/index").fsexportbigquery; const wrapped = functionsTest.wrap(ref); return wrapped(document, data); }; -export const mockedFirestoreBigQueryEventHistoryTracker = () => {}; - -let restoreEnv; -let functionsTest = functionsTestInit(); - describe("extension", () => { beforeEach(() => { restoreEnv = mockedEnv(defaultEnvironment); + jest.resetModules(); + }); + + afterEach(() => { + restoreEnv(); + jest.clearAllMocks(); }); test("functions are exported", () => { @@ -79,21 +75,18 @@ describe("extension", () => { describe("functions.fsexportbigquery", () => { let functionsConfig; - beforeEach(async () => { - jest.resetModules(); - functionsTest = functionsTestInit(); - + beforeEach(() => { functionsConfig = config; }); - test("functions runs with a deletion", async () => { + test("function runs with a CREATE event and publishes both old and new events", async () => { const beforeSnapshot = functionsTest.firestore.makeDocumentSnapshot( - { foo: "bar" }, - "document/path" + {}, // Empty data to simulate no document + "example/doc1" ); const afterSnapshot = functionsTest.firestore.makeDocumentSnapshot( - { foo: "bars" }, - "document/path" + { foo: "bar" }, + "example/doc1" ); const documentChange = functionsTest.makeChange( @@ -102,32 +95,45 @@ describe("extension", () => { ); const callResult = await mockExport(documentChange, { - resource: { - name: "test", - }, + resource: { name: "example/doc1" }, }); expect(callResult).toBeUndefined(); + // Verify Logs expect(mockConsoleLog).toBeCalledWith( "Started execution of extension with configuration", functionsConfig ); + expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); - // sleep for 10 seconds - await new Promise((resolve) => setTimeout(resolve, 10000)); + // Verify Event Publishing + const eventarcMock = require("firebase-admin/eventarc").getEventarc; + const channelMock = eventarcMock().channel(); - expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); - }, 20000); + expect(channelMock.publish).toHaveBeenCalledTimes(2); + expect(channelMock.publish).toHaveBeenCalledWith( + expect.objectContaining({ + type: "firebase.extensions.firestore-counter.v1.onStart", + data: expect.any(Object), + }) + ); + expect(channelMock.publish).toHaveBeenCalledWith( + expect.objectContaining({ + type: "firebase.extensions.firestore-bigquery-export.v1.onStart", + data: expect.any(Object), + }) + ); + }); - test("function runs with updated data", async () => { + test("function runs with a DELETE event", async () => { const beforeSnapshot = functionsTest.firestore.makeDocumentSnapshot( { foo: "bar" }, - "document/path" + "example/doc1" ); const afterSnapshot = functionsTest.firestore.makeDocumentSnapshot( - { foo: "bars" }, - "document/path" + {}, // Empty data to simulate no document + "example/doc1" ); const documentChange = functionsTest.makeChange( @@ -136,19 +142,35 @@ describe("extension", () => { ); const callResult = await mockExport(documentChange, { - resource: { - name: "test", - }, + resource: { name: "example/doc1" }, }); expect(callResult).toBeUndefined(); + // Verify Logs expect(mockConsoleLog).toBeCalledWith( "Started execution of extension with configuration", functionsConfig ); - expect(mockConsoleLog).toBeCalledWith("Completed execution of extension"); + + // Verify Event Publishing for both old and new event types + const eventarcMock = require("firebase-admin/eventarc").getEventarc; + const channelMock = eventarcMock().channel(); + + expect(channelMock.publish).toHaveBeenCalledTimes(2); + expect(channelMock.publish).toHaveBeenCalledWith( + expect.objectContaining({ + type: "firebase.extensions.firestore-counter.v1.onCompletion", + data: expect.any(Object), + }) + ); + expect(channelMock.publish).toHaveBeenCalledWith( + expect.objectContaining({ + type: "firebase.extensions.firestore-bigquery-export.v1.onCompletion", + data: expect.any(Object), + }) + ); }); }); }); diff --git a/firestore-bigquery-export/functions/src/events.ts b/firestore-bigquery-export/functions/src/events.ts index acf14783e..d7506dcea 100644 --- a/firestore-bigquery-export/functions/src/events.ts +++ b/firestore-bigquery-export/functions/src/events.ts @@ -2,14 +2,38 @@ import * as eventArc from "firebase-admin/eventarc"; const { getEventarc } = eventArc; +/** + * The extension name is used to construct event types dynamically. + * Changing this name affects the new event type generation. + * @constant EXTENSION_NAME + */ const EXTENSION_NAME = "firestore-bigquery-export"; -const getEventType = (eventName: string) => - `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`; +/** + * Generates both the OLD and NEW event types to maintain backward compatibility. + * + * Old Event Type: firebase.extensions.firestore-counter.v1.{eventName} + * New Event Type: firebase.extensions.firestore-bigquery-export.v1.{eventName} + * + * @param eventName The name of the event (e.g., "onStart", "onError", etc.) + * @returns An array containing both the old and new event types + */ +const getEventTypes = (eventName: string) => [ + `firebase.extensions.firestore-counter.v1.${eventName}`, // OLD Event Type for backward compatibility + `firebase.extensions.${EXTENSION_NAME}.v1.${eventName}`, // NEW Event Type following the updated convention +]; let eventChannel: eventArc.Channel; -/** setup events */ +/** + * Sets up the Eventarc channel. + * + * This function retrieves the Eventarc channel based on the environment variables: + * - `EVENTARC_CHANNEL` specifies the channel to use for publishing events. + * - `EXT_SELECTED_EVENTS` defines the allowed event types. + * + * @function setupEventChannel + */ export const setupEventChannel = () => { eventChannel = process.env.EVENTARC_CHANNEL && @@ -18,25 +42,60 @@ export const setupEventChannel = () => { }); }; +/** + * Publishes a "start" event using both OLD and NEW event types. + * + * @param data The payload to send with the event. Can be a string or an object. + * @returns A Promise resolving when both events are published. + */ export const recordStartEvent = async (data: string | object) => { if (!eventChannel) return; - return eventChannel.publish({ - type: getEventType("onStart"), - data, - }); + const eventTypes = getEventTypes("onStart"); + + // Publish events for both OLD and NEW event types + return Promise.all( + eventTypes.map((type) => + eventChannel.publish({ + type, + data, + }) + ) + ); }; +/** + * Publishes an "error" event using both OLD and NEW event types. + * + * @param err The Error object containing the error message. + * @param subject (Optional) Subject identifier related to the error event. + * @returns A Promise resolving when both events are published. + */ export const recordErrorEvent = async (err: Error, subject?: string) => { if (!eventChannel) return; - return eventChannel.publish({ - type: getEventType("onError"), - data: { message: err.message }, - subject, - }); + const eventTypes = getEventTypes("onError"); + + // Publish events for both OLD and NEW event types + return Promise.all( + eventTypes.map((type) => + eventChannel.publish({ + type, + data: { message: err.message }, + subject, + }) + ) + ); }; +/** + * Publishes a "success" event using both OLD and NEW event types. + * + * @param params An object containing the subject and the event data. + * @param params.subject A string representing the subject of the event. + * @param params.data The payload to send with the event. + * @returns A Promise resolving when both events are published. + */ export const recordSuccessEvent = async ({ subject, data, @@ -46,18 +105,38 @@ export const recordSuccessEvent = async ({ }) => { if (!eventChannel) return; - return eventChannel.publish({ - type: getEventType("onSuccess"), - subject, - data, - }); + const eventTypes = getEventTypes("onSuccess"); + + // Publish events for both OLD and NEW event types + return Promise.all( + eventTypes.map((type) => + eventChannel.publish({ + type, + subject, + data, + }) + ) + ); }; +/** + * Publishes a "completion" event using both OLD and NEW event types. + * + * @param data The payload to send with the event. Can be a string or an object. + * @returns A Promise resolving when both events are published. + */ export const recordCompletionEvent = async (data: string | object) => { if (!eventChannel) return; - return eventChannel.publish({ - type: getEventType("onCompletion"), - data, - }); + const eventTypes = getEventTypes("onCompletion"); + + // Publish events for both OLD and NEW event types + return Promise.all( + eventTypes.map((type) => + eventChannel.publish({ + type, + data, + }) + ) + ); };