perf(gatsby): enable fast filters for in comparator

packages/gatsby/src/redux/nodes.ts

@@ -4,11 +4,17 @@ import { createPageDependency } from "./actions/add-page-dependency"
 import { IDbQueryElemMatch } from "../db/common/query"
 
 // Only list supported ops here. "CacheableFilterOp"
-type FilterOp = "$eq" | "$ne" | "$lt" | "$lte" | "$gt" | "$gte"
+type FilterOp = "$eq" | "$ne" | "$lt" | "$lte" | "$gt" | "$gte" | "$in"
 // Note: `undefined` is an encoding for a property that does not exist
-type FilterValueNullable = string | number | boolean | null | undefined
+type FilterValueNullable =
+  | string
+  | number
+  | boolean
+  | null
+  | undefined
+  | Array<string | number | boolean | null | undefined>
 // This is filter value in most cases
-type FilterValue = string | number | boolean
+type FilterValue = string | number | boolean | Array<string | number | boolean>
 export type FilterCacheKey = string
 export interface IFilterCache {
   op: FilterOp
@@ -604,6 +610,39 @@ export const getNodesFromCacheByValue = (
     return filterCache.byValue.get(filterValue)
   }
 
+  if (op === `$in`) {
+    if (!Array.isArray(filterValue)) {
+      // Sift assumes the value has an `indexOf` property. By this fluke,
+      // string args would work, but I don't think that's intentional/expected.
+      throw new Error("The argument to the `in` comparator should be an array")
+    }
+    const filterValueArr: Array<FilterValueNullable> = filterValue
+
+    const set = new Set<IGatsbyNode>()
+    if (filterValueArr.includes(null)) {
+      // Like all other ops, `in: [null]` behaves weirdly, allowing all nodes
+      // that do not actually have a (complete) path (v=undefined)
+      const nodes = filterCache.byValue.get(undefined)
+      if (nodes) {
+        nodes.forEach(v => set.add(v))
+      }
+    }
+
+    // For every value in the needle array, find the bucket of nodes for
+    // that value, add this bucket of nodes to one set, return the set.
+    filterValueArr
+      .slice(0) // Sort is inline so slice the original array
+      .sort((a, b) => {
+        if (a == null || b == null) return 0
+        return a < b ? -1 : a > b ? 1 : 0
+      }) // Just sort to preserve legacy order as much as possible.
+      .forEach((v: FilterValueNullable) =>
+        filterCache.byValue.get(v)?.forEach(v => set.add(v))
+      )
+
+    return set
+  }
+
   if (op === `$ne`) {
     const set = new Set(filterCache.meta.nodesUnordered)
 
@@ -635,6 +674,12 @@ export const getNodesFromCacheByValue = (
     return filterCache.byValue.get(filterValue)
   }
 
+  if (Array.isArray(filterValue)) {
+    throw new Error(
+      "Array is an invalid filter value for the `" + op + "` comparator"
+    )
+  }
+
   if (op === `$lt`) {
     // First try a direct approach. If a value is queried that also exists then
     // we can prevent a binary search through the whole set, O(1) vs O(log n)

packages/gatsby/src/redux/run-sift.js

@@ -19,7 +19,7 @@ const {
   getNode: siftGetNode,
 } = require(`./nodes`)
 
-const FAST_OPS = [`$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`]
+const FAST_OPS = [`$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`]
 
 // More of a testing mechanic, to verify whether last runSift call used Sift
 let lastFilterUsedSift = false

packages/gatsby/src/schema/__tests__/connection-filter-on-linked-nodes.js

@@ -285,12 +285,12 @@ describe(`filtering on linked nodes`, () => {
     }
 
     expect(result.data.eq.edges).toEqual([`bar`, `baz`].map(itemToEdge))
-    expect(result.data.in.edges).toEqual([`bar`, `baz`, `foo`].map(itemToEdge))
+    expect(result.data.in.edges).toEqual([`bar`, `foo`, `baz`].map(itemToEdge))
     expect(result.data.insideInlineArrayEq.edges).toEqual(
       [`lorem`, `ipsum`, `sit`].map(itemToEdge)
     )
     expect(result.data.insideInlineArrayIn.edges).toEqual(
-      [`lorem`, `ipsum`, `sit`, `dolor`].map(itemToEdge)
+      [`lorem`, `ipsum`, `dolor`, `sit`].map(itemToEdge)
     )
   })
 

packages/gatsby/src/schema/__tests__/run-query.js

@@ -1182,7 +1182,7 @@ it(`should use the cache argument`, async () => {
       describe(`$in`, () => {
         it(`handles the in operator for strings`, async () => {
           const needle = [`b`, `c`]
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             string: { in: needle },
           })
 
@@ -1197,7 +1197,7 @@ it(`should use the cache argument`, async () => {
 
         it(`handles the in operator for ints`, async () => {
           const needle = [0, 2]
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             index: { in: needle },
           })
 
@@ -1212,7 +1212,7 @@ it(`should use the cache argument`, async () => {
 
         it(`handles the in operator for floats`, async () => {
           const needle = [1.5, 2.5]
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             float: { in: needle },
           })
 
@@ -1226,11 +1226,10 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the in operator for just null`, async () => {
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             nil: { in: [null] },
           })
 
-          // Do not include the nodes without a `nil` property
           // May not have the property, or must be null
           expect(result?.length).toEqual(
             allNodes.filter(node => node.nil === undefined || node.nil === null)
@@ -1243,11 +1242,10 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the in operator for double null`, async () => {
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             nil: { in: [null, null] },
           })
 
-          // Do not include the nodes without a `nil` property
           // May not have the property, or must be null
           expect(result?.length).toEqual(
             allNodes.filter(node => node.nil === undefined || node.nil === null)
@@ -1260,7 +1258,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the in operator for null in int and null`, async () => {
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             nil: { in: [5, null] },
           })
 
@@ -1276,7 +1274,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the in operator for int in int and null`, async () => {
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             index: { in: [2, null] },
           })
 
@@ -1300,7 +1298,7 @@ it(`should use the cache argument`, async () => {
         })
 
         it(`handles the in operator for booleans`, async () => {
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             boolean: { in: [true] },
           })
 
@@ -1313,7 +1311,7 @@ it(`should use the cache argument`, async () => {
 
         it(`handles the in operator for array with one element`, async () => {
           // Note: `node.anArray` doesn't exist or it's an array of multiple numbers
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             anArray: { in: [5] },
           })
 
@@ -1332,7 +1330,7 @@ it(`should use the cache argument`, async () => {
         it(`handles the in operator for array some elements`, async () => {
           // Note: `node.anArray` doesn't exist or it's an array of multiple numbers
           const needle = [20, 5, 300]
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             anArray: { in: needle },
           })
 
@@ -1351,7 +1349,7 @@ it(`should use the cache argument`, async () => {
 
         it(`handles the nested in operator for array of strings`, async () => {
           const needle = [`moo`]
-          const [result, allNodes] = await runSlowFilter({
+          const [result, allNodes] = await runFastFilter({
             frontmatter: { tags: { in: needle } },
           })
 
@@ -1368,6 +1366,31 @@ it(`should use the cache argument`, async () => {
             ).toEqual(true)
           )
         })
+
+        it(`refuses a non-arg number argument`, async () => {
+          await expect(
+            runFastFilter({
+              hair: { in: 2 },
+            })
+          ).rejects.toThrow()
+        })
+
+        // I'm convinced this only works in Sift because of a fluke
+        it.skip(`refuses a non-arg string argument`, async () => {
+          await expect(
+            runFastFilter({
+              name: { in: `The Mad Max` },
+            })
+          ).rejects.toThrow()
+        })
+
+        it(`refuses a non-arg boolean argument`, async () => {
+          await expect(
+            runFastFilter({
+              boolean: { in: true },
+            })
+          ).rejects.toThrow()
+        })
       })
 
       describe(`$elemMatch`, () => {