Improve support for subcollections by propagating dynamic vars and backfilling collectionGroup

extensions/test-params-subcategory-flatten-nested-false.local.env

@@ -0,0 +1,12 @@
+LOCATION=us-central1
+FIRESTORE_COLLECTION_PATH=users/{personId}/books
+TEST_FIRESTORE_PARENT_COLLECTION_PATH=users
+TEST_FIRESTORE_PARENT_ID=123
+TEST_FIRESTORE_CHILD_FIELD_NAME=books
+FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref
+FLATTEN_NESTED_DOCUMENTS=false
+TYPESENSE_HOSTS=localhost
+TYPESENSE_PORT=8108
+TYPESENSE_PROTOCOL=http
+TYPESENSE_COLLECTION_NAME=books_firestore/1
+TYPESENSE_API_KEY=xyz

functions/src/backfill.js

@@ -41,7 +41,17 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
 
   const typesense = createTypesenseClient();
 
-  const querySnapshot = await admin.firestore().collection(config.firestoreCollectionPath);
+  const pathSegments = config.firestoreCollectionPath.split("/").filter(Boolean);
+  const pathPlaceholders = utils.parseFirestorePath(config.firestoreCollectionPath);
+  const isGroupQuery = pathSegments.length > 1 && Object.values(pathPlaceholders).length;
+
+  let querySnapshot;
+  if (isGroupQuery) {
+    const collectionGroup = pathSegments.pop();
+    querySnapshot = admin.firestore().collectionGroup(collectionGroup);
+  } else {
+    querySnapshot = admin.firestore().collection(config.firestoreCollectionPath);
+  }
 
   let lastDoc = null;
 
@@ -53,9 +63,15 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
     }
     const currentDocumentsBatch = await Promise.all(
         thisBatch.docs.map(async (doc) => {
-          return await utils.typesenseDocumentFromSnapshot(doc);
-        }),
-    );
+          const docPath = doc.ref.path;
+          const pathParams = utils.pathMatchesSelector(docPath, config.firestoreCollectionPath);
+
+          if (!isGroupQuery || (isGroupQuery && pathParams !== null)) {
+            return await utils.typesenseDocumentFromSnapshot(doc, pathParams);
+          } else {
+            return null;
+          }
+        }).filter((doc) => doc !== null));
 
     lastDoc = thisBatch.docs.at(-1) ?? null;
     try {

functions/src/indexOnWrite.js

@@ -20,7 +20,7 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
 
         // snapshot.after.ref.get() will refetch the latest version of the document
         const latestSnapshot = await snapshot.after.ref.get();
-        const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot);
+        const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, context.params);
 
         functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
         return await typesense

functions/src/utils.js

@@ -131,10 +131,11 @@ function flattenDocument(obj, prefix = "") {
 
 /**
  * @param {DocumentSnapshot} firestoreDocumentSnapshot
+ * @param {Object} contextParams
  * @param {Array} fieldsToExtract
  * @return {Object} typesenseDocument
  */
-exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, fieldsToExtract = config.firestoreCollectionFields) => {
+exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, contextParams = {}, fieldsToExtract = config.firestoreCollectionFields) => {
   const data = firestoreDocumentSnapshot.data();
 
   const extractedData = fieldsToExtract.length === 0 ? data : fieldsToExtract.reduce((acc, field) => extractField(data, acc, field), {});
@@ -148,5 +149,77 @@ exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, fields
 
   typesenseDocument.id = firestoreDocumentSnapshot.id;
 
+  if (contextParams && Object.entries(contextParams).length) {
+    Object.entries(contextParams).forEach(([key, value]) => {
+      typesenseDocument[key] = value;
+    });
+  }
+
   return typesenseDocument;
 };
+
+/**
+ * Parses a Firestore path with placeholdersto extract indices and names of placeholders.
+ * @param {string} firestorePath - The Firestore path to parse.
+ * @return {Object} An object containing the names of placeholders, and their corresponding indices.
+ * @throws Will throw an error if the path is invalid.
+ */
+exports.parseFirestorePath = function(firestorePath) {
+  if (!firestorePath || typeof firestorePath !== "string") {
+    throw new Error("Invalid Firestore path: Path must be a non-empty string.");
+  }
+
+  const segments = firestorePath.split("/").filter(Boolean);
+  const placeholders = {};
+
+  segments.forEach((segment, index) => {
+    const match = segment.match(/^{([^}]+)}$/); // Match placeholders like {userId}
+    if (match) {
+      const varName = match[1];
+      if (placeholders[varName]) {
+        throw new Error(`Duplicate placeholder detected: ${varName}`);
+      }
+      placeholders[varName] = index;
+    }
+  });
+
+  return placeholders;
+};
+
+/**
+ * Verifies if a given Firestore path matches a selector and extracts placeholder values.
+ * @param {string} path - The static Firestore path (e.g., "users/123/library/456/books/789").
+ * @param {string} selector - The path selector with placeholders (e.g., "users/{userId}/library/{libraryId}/books").
+ * @return {Object|null} A dictionary of extracted values if the path matches the selector, or `null` if it does not match.
+ */
+exports.pathMatchesSelector = function(path, selector) {
+  if (!path || typeof path !== "string") {
+    throw new Error("Invalid path: Path must be a non-empty string.");
+  }
+  if (!selector || typeof selector !== "string") {
+    throw new Error("Invalid selector: Selector must be a non-empty string.");
+  }
+
+  const pathSegments = path.split("/").filter(Boolean);
+  const selectorSegments = selector.split("/").filter(Boolean);
+
+  if (pathSegments.length < selectorSegments.length) {
+    return null;
+  }
+
+  const extractedValues = {};
+
+  for (let i = 0; i < selectorSegments.length; i++) {
+    const selectorSegment = selectorSegments[i];
+    const pathSegment = pathSegments[i];
+
+    if (selectorSegment.startsWith("{") && selectorSegment.endsWith("}")) {
+      const placeholderName = selectorSegment.slice(1, -1); // Remove {}
+      extractedValues[placeholderName] = pathSegment;
+    } else if (selectorSegment !== pathSegment) {
+      return null;
+    }
+  }
+
+  return extractedValues;
+};

package.json

@@ -4,9 +4,10 @@
   "scripts": {
     "emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
     "export": "firebase emulators:export emulator_data",
-    "test": "npm run test-part-1 && npm run test-part-2",
-    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\"'",
+    "test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3",
+    "test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\" --testPathIgnorePatterns=\"Subcollection\"'",
     "test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
+    "test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'",
     "typesenseServer": "docker compose up",
     "lint:fix": "eslint . --fix",
     "lint": "eslint ."