Add support for Firestore TTL (#5267)

* Add support for Firestore TTL * Add support for Firestore TTL * Remove unnecessary TTL check * Remove unnecessary TTL check * Add support for Firestore TTL
firebase · Dec 6, 2022 · 27def0a · 27def0a
1 parent f83fc6e
commit 27def0a
Show file tree

Hide file tree

Showing 9 changed files with 230 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1 @@
+- Add support for Firestore TTL (#5267)
diff --git a/src/firestore/README.md b/src/firestore/README.md
@@ -61,9 +61,12 @@ The schema for one object in the `fieldOverrides` array is as follows. Optional
 
 Note that Cloud Firestore document fields can only be indexed in one [mode](https://firebase.google.com/docs/firestore/query-data/index-overview#index_modes), thus a field object cannot contain both the `order` and `arrayConfig` properties.
 
+For more information about time-to-live (TTL) policies review the [official documention](https://cloud.google.com/firestore/docs/ttl).
+
 ```javascript
   collectionGroup: string  // Labeled "Collection ID" in the Firebase console
   fieldPath: string
+  ttl?: boolean            // Set specified field to have TTL policy and be eligible for deletion
   indexes: array           // Set empty array to disable indexes on this collectionGroup + fieldPath
     queryScope: string     // One of "COLLECTION", "COLLECTION_GROUP"
     order?: string         // One of "ASCENDING", "DESCENDING"; excludes arrayConfig property

diff --git a/src/firestore/indexes-api.ts b/src/firestore/indexes-api.ts
@@ -30,6 +30,12 @@ export enum State {
   NEEDS_REPAIR = "NEEDS_REPAIR",
 }
 
+export enum StateTtl {
+  CREATING = "CREATING",
+  ACTIVE = "ACTIVE",
+  NEEDS_REPAIR = "NEEDS_REPAIR",
+}
+
 /**
  * An Index as it is represented in the Firestore v1beta2 indexes API.
  */
@@ -49,6 +55,13 @@ export interface IndexField {
   arrayConfig?: ArrayConfig;
 }
 
+/**
+ * TTL policy configuration for a field
+ */
+export interface TtlConfig {
+  state: StateTtl;
+}
+
 /**
  * Represents a single field in the database.
  *
@@ -58,6 +71,7 @@ export interface IndexField {
 export interface Field {
   name: string;
   indexConfig: IndexConfig;
+  ttlConfig?: TtlConfig;
 }
 
 /**

diff --git a/src/firestore/indexes-sort.ts b/src/firestore/indexes-sort.ts
@@ -92,13 +92,21 @@ export function compareApiField(a: API.Field, b: API.Field): number {
  * Comparisons:
  *   1) The collection group.
  *   2) The field path.
+ *   3) The ttl.
  *   3) The list of indexes.
  */
 export function compareFieldOverride(a: Spec.FieldOverride, b: Spec.FieldOverride): number {
   if (a.collectionGroup !== b.collectionGroup) {
     return a.collectionGroup.localeCompare(b.collectionGroup);
   }
 
+  // The ttl override can be undefined, we only guarantee that true values will
+  // come last since those overrides should be executed after disabling TTL per collection.
+  const compareTtl = Number(!!a.ttl) - Number(!!b.ttl);
+  if (compareTtl) {
+    return compareTtl;
+  }
+
   if (a.fieldPath !== b.fieldPath) {
     return a.fieldPath.localeCompare(b.fieldPath);
   }

diff --git a/src/firestore/indexes-spec.ts b/src/firestore/indexes-spec.ts
@@ -21,6 +21,7 @@ export interface Index {
 export interface FieldOverride {
   collectionGroup: string;
   fieldPath: string;
+  ttl?: boolean;
   indexes: FieldIndex[];
 }
 

diff --git a/src/firestore/indexes.ts b/src/firestore/indexes.ts
@@ -140,7 +140,10 @@ export class FirestoreIndexes {
       }
     }
 
-    for (const field of fieldOverridesToDeploy) {
+    // Disabling TTL must be executed first in case another field is enabled for
+    // the same collection in the same deployment.
+    const sortedFieldOverridesToDeploy = fieldOverridesToDeploy.sort(sort.compareFieldOverride);
+    for (const field of sortedFieldOverridesToDeploy) {
       const exists = existingFieldOverrides.some((x) => this.fieldMatchesSpec(x, field));
       if (exists) {
         logger.debug(`Skipping existing field override: ${JSON.stringify(field)}`);
@@ -195,7 +198,7 @@ export class FirestoreIndexes {
    */
   async listFieldOverrides(project: string): Promise<API.Field[]> {
     const parent = `projects/${project}/databases/(default)/collectionGroups/-`;
-    const url = `/${parent}/fields?filter=indexConfig.usesAncestorConfig=false`;
+    const url = `/${parent}/fields?filter=indexConfig.usesAncestorConfig=false OR ttlConfig:*`;
 
     const res = await this.apiClient.get<{ fields?: API.Field[] }>(url);
     const fields = res.body.fields;
@@ -236,6 +239,7 @@ export class FirestoreIndexes {
       return {
         collectionGroup: parsedName.collectionGroupId,
         fieldPath: parsedName.fieldPath,
+        ttl: !!field.ttlConfig,
 
         indexes: fieldIndexes.map((index) => {
           const firstField = index.fields[0];
@@ -339,6 +343,10 @@ export class FirestoreIndexes {
     validator.assertHas(field, "fieldPath");
     validator.assertHas(field, "indexes");
 
+    if (typeof field.ttl !== "undefined") {
+      validator.assertType("ttl", field.ttl, "boolean");
+    }
+
     field.indexes.forEach((index: any) => {
       validator.assertHasOneOf(index, ["arrayConfig", "order"]);
 
@@ -379,23 +387,33 @@ export class FirestoreIndexes {
       };
     });
 
-    const data = {
+    let data = {
       indexConfig: {
         indexes,
       },
     };
 
-    await this.apiClient.patch(url, data);
+    if (spec.ttl) {
+      data = Object.assign(data, {
+        ttlConfig: {},
+      });
+    }
+
+    if (typeof spec.ttl !== "undefined") {
+      await this.apiClient.patch(url, data);
+    } else {
+      await this.apiClient.patch(url, data, { queryParams: { updateMask: "indexConfig" } });
+    }
   }
 
   /**
-   * Delete an existing index on the specified project.
+   * Delete an existing field overrides on the specified project.
    */
   deleteField(field: API.Field): Promise<any> {
     const url = field.name;
     const data = {};
 
-    return this.apiClient.patch(`/${url}`, data, { queryParams: { updateMask: "indexConfig" } });
+    return this.apiClient.patch(`/${url}`, data);
   }
 
   /**
@@ -471,6 +489,16 @@ export class FirestoreIndexes {
       return false;
     }
 
+    if (typeof spec.ttl !== "undefined" && util.booleanXOR(!!field.ttlConfig, spec.ttl)) {
+      return false;
+    } else if (!!field.ttlConfig && typeof spec.ttl === "undefined") {
+      utils.logLabeledBullet(
+        "firestore",
+        `there are TTL field overrides for collection ${spec.collectionGroup} defined in your project that are not present in your ` +
+          "firestore indexes file. The TTL policy won't be deleted since is not specified as false."
+      );
+    }
+
     const fieldIndexes = field.indexConfig.indexes || [];
     if (fieldIndexes.length !== spec.indexes.length) {
       return false;
@@ -619,6 +647,10 @@ export class FirestoreIndexes {
     } else {
       result += " (no indexes)";
     }
+    const fieldTtl = field.ttlConfig;
+    if (fieldTtl) {
+      result += ` TTL(${fieldTtl.state})`;
+    }
 
     return result;
   }

diff --git a/src/firestore/util.ts b/src/firestore/util.ts
@@ -55,3 +55,10 @@ export function parseFieldName(name: string): FieldName {
     fieldPath: m[3],
   };
 }
+
+/**
+ * Performs XOR operator between two boolean values
+ */
+export function booleanXOR(a: boolean, b: boolean): boolean {
+  return !!(Number(a) - Number(b));
+}
diff --git a/src/firestore/validator.ts b/src/firestore/validator.ts
@@ -39,3 +39,13 @@ export function assertEnum(obj: any, prop: string, valid: any[]): void {
     throw new FirebaseError(`Field "${prop}" must be one of  ${valid.join(", ")}: ${objString}`);
   }
 }
+
+/**
+ * Throw an error if the value of the property 'prop' differs against type
+ * guard.
+ */
+export function assertType(prop: string, propValue: any, type: string): void {
+  if (typeof propValue !== type) {
+    throw new FirebaseError(`Property "${prop}" must be of type ${type}`);
+  }
+}
diff --git a/src/test/firestore/indexes.spec.ts b/src/test/firestore/indexes.spec.ts
@@ -200,6 +200,85 @@ describe("IndexSpecMatching", () => {
     expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
   });
 
+  it("should identify a positive field spec match with ttl specified as false", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123",
+      indexConfig: {
+        indexes: [
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", order: "ASCENDING" }],
+          },
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "abc123", arrayConfig: "CONTAINS" }],
+          },
+        ],
+      },
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "abc123",
+      ttl: false,
+      indexes: [
+        { order: "ASCENDING", queryScope: "COLLECTION" },
+        { arrayConfig: "CONTAINS", queryScope: "COLLECTION" },
+      ],
+    } as Spec.FieldOverride;
+
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
+  });
+
+  it("should identify a positive ttl field spec match", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/fieldTtl",
+      indexConfig: {
+        indexes: [
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "fieldTtl", order: "ASCENDING" }],
+          },
+        ],
+      },
+      ttlConfig: {
+        state: "ACTIVE",
+      },
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "fieldTtl",
+      ttl: true,
+      indexes: [{ order: "ASCENDING", queryScope: "COLLECTION" }],
+    } as Spec.FieldOverride;
+
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
+  });
+
+  it("should identify a negative ttl field spec match", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/fieldTtl",
+      indexConfig: {
+        indexes: [
+          {
+            queryScope: "COLLECTION",
+            fields: [{ fieldPath: "fieldTtl", order: "ASCENDING" }],
+          },
+        ],
+      },
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "fieldTtl",
+      ttl: true,
+      indexes: [{ order: "ASCENDING", queryScope: "COLLECTION" }],
+    } as Spec.FieldOverride;
+
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(false);
+  });
+
   it("should match a field spec with all indexes excluded", () => {
     const apiField = {
       name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123",
@@ -215,6 +294,25 @@ describe("IndexSpecMatching", () => {
     expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
   });
 
+  it("should match a field spec with only ttl", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/ttlField",
+      ttlConfig: {
+        state: "ACTIVE",
+      },
+      indexConfig: {},
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "ttlField",
+      ttl: true,
+      indexes: [],
+    } as Spec.FieldOverride;
+
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(true);
+  });
+
   it("should identify a negative field spec match", () => {
     const apiField = {
       name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/abc123",
@@ -244,6 +342,27 @@ describe("IndexSpecMatching", () => {
     // The second spec contains "DESCENDING" where the first contains "ASCENDING"
     expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(false);
   });
+
+  it("should identify a negative field spec match with ttl as false", () => {
+    const apiField = {
+      name: "/projects/myproject/databases/(default)/collectionGroups/collection/fields/fieldTtl",
+      ttlConfig: {
+        state: "ACTIVE",
+      },
+      indexConfig: {},
+    } as API.Field;
+
+    const specField = {
+      collectionGroup: "collection",
+      fieldPath: "fieldTtl",
+      ttl: false,
+      indexes: [],
+    } as Spec.FieldOverride;
+
+    // The second spec contains "false" for ttl  where the first contains "true"
+    // for ttl
+    expect(idx.fieldMatchesSpec(apiField, specField)).to.eql(false);
+  });
 });
 
 describe("IndexSorting", () => {
@@ -360,6 +479,35 @@ describe("IndexSorting", () => {
     expect([b, a, d, c].sort(sort.compareFieldOverride)).to.eql([a, b, c, d]);
   });
 
+  it("should sort ttl true to be last in an array of Spec field overrides", () => {
+    // Sorts first because of collectionGroup
+    const a: Spec.FieldOverride = {
+      collectionGroup: "collectionA",
+      fieldPath: "fieldA",
+      ttl: false,
+      indexes: [],
+    };
+    const b: Spec.FieldOverride = {
+      collectionGroup: "collectionA",
+      fieldPath: "fieldB",
+      ttl: true,
+      indexes: [],
+    };
+    const c: Spec.FieldOverride = {
+      collectionGroup: "collectionB",
+      fieldPath: "fieldA",
+      ttl: false,
+      indexes: [],
+    };
+    const d: Spec.FieldOverride = {
+      collectionGroup: "collectionB",
+      fieldPath: "fieldB",
+      ttl: true,
+      indexes: [],
+    };
+    expect([b, a, d, c].sort(sort.compareFieldOverride)).to.eql([a, b, c, d]);
+  });
+
   it("should correctly sort an array of API indexes", () => {
     // Sorts first because of collectionGroup
     const a: API.Index = {