From e3fdd60b978d95cb20f515d8b652915472155bb9 Mon Sep 17 00:00:00 2001 From: James Sasitorn Date: Sun, 15 Dec 2024 21:54:59 -0800 Subject: [PATCH] Add test cases for subcollections for backfill --- test/backfillSubcollection.spec.js | 311 +++++++++++++++++++++++++++++ test/utilsSubcollection.js | 109 ++++++++++ 2 files changed, 420 insertions(+) create mode 100644 test/backfillSubcollection.spec.js create mode 100644 test/utilsSubcollection.js diff --git a/test/backfillSubcollection.spec.js b/test/backfillSubcollection.spec.js new file mode 100644 index 0000000..b13870e --- /dev/null +++ b/test/backfillSubcollection.spec.js @@ -0,0 +1,311 @@ +const firebase = require("firebase-admin"); +const config = require("../functions/src/config.js"); +const typesense = require("../functions/src/createTypesenseClient.js")(); + +const app = firebase.initializeApp({ + // Use a special URL to talk to the Realtime Database emulator + databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`, + projectId: process.env.GCLOUD_PROJECT, +}); +const firestore = app.firestore(); + +describe("backfillSubcollection", () => { + const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH; + const unrelatedCollectionPath = "unrelatedCollectionToNotBackfill"; + const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME; + let parentIdField = null; + + beforeAll(() => { + const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g); + expect(matches).toBeDefined(); + expect(matches.length).toBe(1); + + parentIdField = matches[0].replace(/{|}/g, ""); + expect(parentIdField).toBeDefined(); + }); + + beforeEach(async () => { + // Clear the database between tests + await firestore.recursiveDelete(firestore.collection(parentCollectionPath)); + await firestore.recursiveDelete(firestore.collection(unrelatedCollectionPath)); + + // Clear any previously created collections + try { + await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + } catch (e) { + console.info(`${config.typesenseCollectionName} collection not found, proceeding...`); + } + + // Create a new Typesense collection + return typesense.collections().create({ + name: config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + }); + + afterAll(async () => { + // clean up the firebase app after all tests have run + await app.delete(); + }); + + describe("when firestore_collections is not specified", () => { + it("backfills existing Firestore data in all collections to Typesense" + + " when `trigger: true` is set " + + ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + const parentDocData = { + nested_field: { + field1: "value1", + }, + }; + + const subDocData = { + author: "Author A", + title: "Title X", + country: "USA", + }; + + // create parent document in Firestore + const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + + // create a subcollection with document under the parent document + const subCollectionRef = parentDocRef.collection(childFieldName); + const subDocRef = await subCollectionRef.add(subDocData); + + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections().create({ + name: config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await firestore + .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({trigger: true}); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was backfilled + const typesenseDocsStr = await typesense + .collections(encodeURIComponent(config.typesenseCollectionName)) + .documents() + .export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + + expect(typesenseDocs[0]).toStrictEqual({ + id: subDocRef.id, + author: subDocData.author, + title: subDocData.title, + [parentIdField]: parentDocRef.id, + }); + }); + }); + + describe("when firestore_collections is specified", () => { + describe("when firestore_collections includes this collection", () => { + it("backfills existing Firestore data in this particular collection to Typesense" + + " when `trigger: true` is set " + + ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + const parentDocData = { + nested_field: { + field1: "value1", + }, + }; + + const subDocData = { + author: "Author A", + title: "Title X", + country: "USA", + }; + + // create parent document in Firestore + const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + + // create a subcollection with document under the parent document + const subCollectionRef = parentDocRef.collection(childFieldName); + const subDocRef = await subCollectionRef.add(subDocData); + + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections().create({ + name: config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await firestore + .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({ + trigger: true, + firestore_collections: [config.firestoreCollectionPath], + }); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was backfilled + const typesenseDocsStr = await typesense + .collections(encodeURIComponent(config.typesenseCollectionName)) + .documents() + .export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + expect(typesenseDocs[0]).toStrictEqual({ + id: subDocRef.id, + author: subDocData.author, + title: subDocData.title, + [parentIdField]: parentDocRef.id, + }); + }); + }); + + describe("when firestore_collections does not include this collection", () => { + it("does not backfill existing Firestore data in this particular collection to Typesense" + + " when `trigger: true` is set " + + ` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => { + const parentDocData = { + nested_field: { + field1: "value1", + }, + }; + + const subDocData = { + author: "Author A", + title: "Title X", + country: "USA", + }; + + // create parent document in Firestore + const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + + // create a subcollection with document under the parent document + const subCollectionRef = parentDocRef.collection(childFieldName); + await subCollectionRef.add(subDocData); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections().create({ + name: config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await firestore + .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({ + trigger: true, + firestore_collections: ["some/other/collection"], + }); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was not backfilled + const typesenseDocsStr = await typesense + .collections(encodeURIComponent(config.typesenseCollectionName)) + .documents() + .export(); + expect(typesenseDocsStr).toEqual(""); + }); + }); + }); + + describe("Backfill subcollections", () => { + it("Ensure backfill doesnt backfill unrelated collections", async () => { + const parentDocData = { + nested_field: { + field1: "value1", + }, + }; + + const subDocData = { + author: "Author A", + title: "Title X", + country: "USA", + }; + + // create parent document in Firestore + const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData); + + // create a subcollection with document under the parent document + const subCollectionRef = parentDocRef.collection(childFieldName); + const subDocRef = await subCollectionRef.add(subDocData); + + // Create an unrelated set of documents that should not be backfilled + const unrelatedParentDocData = { + nested_field: { + field1: "value3", + }, + }; + + const unrelatedSubDocData = { + author: "Author C", + title: "Title Z", + country: "CAN", + }; + + // create unrelated parent document in Firestore + const unrelatedParentDocRef = await firestore.collection(unrelatedCollectionPath).add(unrelatedParentDocData); + + // create a subcollection with document under the unrelatedparent document + const unrelatedSubCollectionRef = unrelatedParentDocRef.collection(childFieldName); + await unrelatedSubCollectionRef.add(unrelatedSubDocData); + + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // The above will automatically add the document to Typesense, + // so delete it so we can test backfill + await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete(); + await typesense.collections().create({ + name: config.typesenseCollectionName, + fields: [ + {name: ".*", type: "auto"}, + ], + }); + + await firestore + .collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0]) + .doc("backfill") + .set({ + trigger: true, + firestore_collections: [config.firestoreCollectionPath], + }); + // Wait for firestore cloud function to write to Typesense + await new Promise((r) => setTimeout(r, 2000)); + + // Check that the data was backfilled + const typesenseDocsStr = await typesense + .collections(encodeURIComponent(config.typesenseCollectionName)) + .documents() + .export(); + const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s)); + expect(typesenseDocs.length).toBe(1); + + expect(typesenseDocs[0]).toStrictEqual({ + id: subDocRef.id, + author: subDocData.author, + title: subDocData.title, + [parentIdField]: parentDocRef.id, + }); + }); + }); +}); diff --git a/test/utilsSubcollection.js b/test/utilsSubcollection.js new file mode 100644 index 0000000..073e166 --- /dev/null +++ b/test/utilsSubcollection.js @@ -0,0 +1,109 @@ +const utils = require("../functions/src/utils"); + +describe("Utils", () => { + describe("Parsing static firestore path", () => { + it("Static firestore path should do nothing", async () => { + const staticPathVars1 = utils.parseFirestorePath("books"); + expect(staticPathVars1).toEqual({}); + + const staticPathVars2 = utils.parseFirestorePath("books/123"); + expect(staticPathVars2).toEqual({}); + + const staticPathVars5 = utils.parseFirestorePath("books/123/chapter/456/title"); + expect(staticPathVars5).toEqual({}); + }); + + it("Throws an exception if placeholders are empty but fullPath has segments", async () => { + const fullPath = "/users/123/books/456/author"; + const selector = "/users/123/books/456/author/{authorId}/email"; + + const vars = utils.pathMatchesSelector(fullPath, selector); + expect(vars).toBeNull(); + }); + }); + + describe("Parsing dynamic firestore path", () => { + it("Dynamic firestore path should return placeholders", async () => { + const dynamicPath2 = "books/{bookId}/chapter"; + const dynamicPathVars2 = utils.parseFirestorePath(dynamicPath2); + expect(dynamicPathVars2).toEqual({bookId: 1}); + + const actualPathVars2 = utils.pathMatchesSelector("books/123/chapter", dynamicPath2); + expect(actualPathVars2).toEqual({bookId: "123"}); + + const dynamicPath5 = "books/{bookId}/chapter/{chapterId}/title"; + const dynamicPathVars5 = utils.parseFirestorePath(dynamicPath5); + expect(dynamicPathVars5).toEqual({bookId: 1, chapterId: 3}); + + const actualPathVars5 = utils.pathMatchesSelector("books/123/chapter/456/title", dynamicPath5); + expect(actualPathVars5).toEqual({bookId: "123", chapterId: "456"}); + }); + }); + + describe("pathMatchesSelector", () => { + it("should return extracted placeholders for a matching path and selector", () => { + const path = "users/123/library/456/books/789"; + const selector = "users/{userId}/library/{libraryId}/books"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toEqual({userId: "123", libraryId: "456"}); + }); + + it("should return null for a non-matching path and selector", () => { + const path = "users/123/library/456/magazines/789"; + const selector = "users/{userId}/library/{libraryId}/books/{bookId}"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toBeNull(); + }); + + it("should return null if the path is shorter than the selector", () => { + const path = "users/123/library/456"; + const selector = "users/{userId}/library/{libraryId}/books/{bookId}"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toBeNull(); + }); + + it("should handle selectors without placeholders", () => { + const path = "users/123/library"; + const selector = "users/123/library"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toEqual({}); + }); + + it("should return null if the static segments of path and selector do not match", () => { + const path = "users/123/libraries/456"; + const selector = "users/{userId}/library/{libraryId}"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toBeNull(); + }); + + it("should throw an error for an invalid path", () => { + const path = null; + const selector = "users/{userId}/library/{libraryId}"; + expect(() => { + utils.pathMatchesSelector(path, selector); + }).toThrow("Invalid path: Path must be a non-empty string."); + }); + + it("should throw an error for an invalid selector", () => { + const path = "users/123/library/456"; + const selector = null; + expect(() => { + utils.pathMatchesSelector(path, selector); + }).toThrow("Invalid selector: Selector must be a non-empty string."); + }); + + it("should extract placeholders even when additional path segments exist", () => { + const path = "users/123/library/456/books/789/reviews/101"; + const selector = "users/{userId}/library/{libraryId}/books"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toEqual({userId: "123", libraryId: "456"}); + }); + + it("should handle paths and selectors with trailing slashes", () => { + const path = "users/123/library/456/"; + const selector = "users/{userId}/library"; + const result = utils.pathMatchesSelector(path, selector); + expect(result).toEqual({userId: "123"}); + }); + }); +});