perf(gatsby): support empty filters without sift (gatsbyjs#24258)

pvdz · web-flow · commit 542cda81f97d · 2020-05-22T12:05:06.000+02:00
* perf(gatsby): support empty filters without sift

* Add a test case for empty filters

* Make return type consistent with what Sift returned

In particular the fast filters were returning `undefined` where they should be returning `null`. Fairly benign internal change but will be relevant once this file gets the TS treatment.

* Remove some old support checks now that all ops are supported

* Only set the resolved property if something got resolved
diff --git a/packages/gatsby/src/redux/nodes.ts b/packages/gatsby/src/redux/nodes.ts
@@ -354,6 +354,58 @@ export const ensureIndexByQuery = (
   postIndexingMetaSetup(filterCache, op)
 }
 
+export function ensureEmptyFilterCache(
+  filterCacheKey,
+  nodeTypeNames: string[],
+  filtersCache: FiltersCache
+): void {
+  // This is called for queries without any filters
+  // We want to cache the result since it's basically a set of nodes by type(s)
+  // There are sites that have multiple queries which are empty
+
+  const state = store.getState()
+  const resolvedNodesCache = state.resolvedNodesCache
+  const nodesUnordered: Array<IGatsbyNode> = []
+
+  filtersCache.set(filterCacheKey, {
+    op: `$eq`, // Ignore.
+    byValue: new Map<FilterValueNullable, Set<IGatsbyNode>>(),
+    meta: {
+      nodesUnordered, // This is what we want
+    },
+  })
+
+  if (nodeTypeNames.length === 1) {
+    getNodesByType(nodeTypeNames[0]).forEach(node => {
+      if (!node.__gatsby_resolved) {
+        const typeName = node.internal.type
+        const resolvedNodes = resolvedNodesCache.get(typeName)
+        const resolved = resolvedNodes?.get(node.id)
+        if (resolved !== undefined) {
+          node.__gatsby_resolved = resolved
+        }
+      }
+      nodesUnordered.push(node)
+    })
+  } else {
+    // Here we must first filter for the node type
+    // This loop is expensive at scale (!)
+    state.nodes.forEach(node => {
+      if (nodeTypeNames.includes(node.internal.type)) {
+        if (!node.__gatsby_resolved) {
+          const typeName = node.internal.type
+          const resolvedNodes = resolvedNodesCache.get(typeName)
+          const resolved = resolvedNodes?.get(node.id)
+          if (resolved !== undefined) {
+            node.__gatsby_resolved = resolved
+          }
+        }
+        nodesUnordered.push(node)
+      }
+    })
+  }
+}
+
 function addNodeToFilterCache(
   node: IGatsbyNode,
   chain: Array<string>,
diff --git a/packages/gatsby/src/redux/run-sift.js b/packages/gatsby/src/redux/run-sift.js
@@ -12,33 +12,22 @@ const {
   dbQueryToSiftQuery,
 } = require(`../db/common/query`)
 const {
+  ensureEmptyFilterCache,
   ensureIndexByQuery,
   ensureIndexByElemMatch,
   getNodesFromCacheByValue,
   addResolvedNodes,
   getNode: siftGetNode,
 } = require(`./nodes`)
 
-const FAST_OPS = [
-  `$eq`,
-  `$ne`,
-  `$lt`,
-  `$lte`,
-  `$gt`,
-  `$gte`,
-  `$in`,
-  `$nin`,
-  `$regex`, // Note: this includes $glob
-]
-
 // More of a testing mechanic, to verify whether last runSift call used Sift
 let lastFilterUsedSift = false
 
 /**
  * Creates a key for one filterCache inside FiltersCache
  *
  * @param {Array<string>} typeNames
- * @param {DbQuery} filter
+ * @param {DbQuery | null} filter If null the key will have empty path/op parts
  * @returns {FilterCacheKey} (a string: `types.join()/path.join()/operator` )
  */
 const createFilterCacheKey = (typeNames, filter) => {
@@ -147,10 +136,15 @@ function handleMany(siftArgs, nodes) {
  *
  * @param {Array<DbQuery>} filters Resolved. (Should be checked by caller to exist)
  * @param {Array<string>} nodeTypeNames
- * @param {FiltersCache} filtersCache
- * @returns {Array<IGatsbyNode> | undefined}
+ * @param {undefined | null | FiltersCache} filtersCache
+ * @returns {Array<IGatsbyNode> | null}
  */
-const runFiltersWithoutSift = (filters, nodeTypeNames, filtersCache) => {
+const filterWithoutSift = (filters, nodeTypeNames, filtersCache) => {
+  if (!filtersCache) {
+    // If no filter cache is passed on, explicitly don't use one
+    return null
+  }
+
   const nodesPerValueSets /*: Array<Set<IGatsbyNode>> */ = getBucketsForFilters(
     filters,
     nodeTypeNames,
@@ -159,7 +153,7 @@ const runFiltersWithoutSift = (filters, nodeTypeNames, filtersCache) => {
 
   if (!nodesPerValueSets) {
     // Let Sift take over as fallback
-    return undefined
+    return null
   }
 
   // Put smallest last (we'll pop it)
@@ -188,11 +182,14 @@ const runFiltersWithoutSift = (filters, nodeTypeNames, filtersCache) => {
   // case for all value pairs? How likely is that to ever be reused?
 
   if (result.length === 0) {
-    return undefined
+    return null
   }
   return result
 }
 
+// Not a public API
+exports.filterWithoutSift = filterWithoutSift
+
 /**
  * @param {Array<DbQuery>} filters
  * @param {Array<string>} nodeTypeNames
@@ -256,11 +253,7 @@ const getBucketsForQueryFilter = (
 ) => {
   let {
     path: filterPath,
-    query: {
-      // Note: comparator is verified to be a FilterOp in filterWithoutSift
-      comparator /*: as FilterOp*/,
-      value: filterValue,
-    },
+    query: { comparator /*: as FilterOp*/, value: filterValue },
   } = filter
 
   if (!filtersCache.has(filterCacheKey)) {
@@ -323,10 +316,6 @@ const collectBucketForElemMatch = (
     }
   }
 
-  if (!FAST_OPS.includes(comparator)) {
-    return false
-  }
-
   if (!filtersCache.has(filterCacheKey)) {
     ensureIndexByElemMatch(
       comparator,
@@ -411,7 +400,7 @@ exports.didLastFilterUseSift = function _didLastFilterUseSift() {
  * @param {Array<string>} nodeTypeNames
  * @param {undefined | null | FiltersCache} filtersCache
  * @param resolvedFields
- * @returns {Array<IGatsbyNode> | undefined} Collection of results. Collection
+ * @returns {Array<IGatsbyNode> | null} Collection of results. Collection
  *   will be limited to 1 if `firstOnly` is true
  */
 const applyFilters = (
@@ -444,7 +433,27 @@ const applyFilters = (
     }
   }
 
-  const result = filterWithoutSift(filters, nodeTypeNames, filtersCache)
+  if (filtersCache && filters.length === 0) {
+    let filterCacheKey = createFilterCacheKey(nodeTypeNames, null)
+    if (!filtersCache.has(filterCacheKey)) {
+      ensureEmptyFilterCache(filterCacheKey, nodeTypeNames, filtersCache)
+    }
+
+    const cache = filtersCache.get(filterCacheKey).meta.nodesUnordered
+
+    lastFilterUsedSift = false
+
+    if (firstOnly || cache.length) {
+      return cache.slice(0)
+    }
+    return null
+  }
+
+  const result /*: Array<IGatsbyNode> | null */ = filterWithoutSift(
+    filters,
+    nodeTypeNames,
+    filtersCache
+  )
 
   lastFilterUsedSift = false
   if (result) {
@@ -458,7 +467,7 @@ const applyFilters = (
   }
   lastFilterUsedSift = true
 
-  const siftResult = filterWithSift(
+  const siftResult /*: Array<IGatsbyNode> | null */ = filterWithSift(
     filters,
     firstOnly,
     nodeTypeNames,
@@ -493,54 +502,14 @@ const filterToStats = (
   }
 }
 
-/**
- * Check if filter op is supported (not all are). If so, uses custom
- * fast indexes based on filter and types and returns any result it finds.
- * If conditions are not met or no nodes are found, returns undefined and
- * a slow run through Sift is executed instead.
- * This function is a noop if no filter cache is given to it.
- *
- * @param {Array<DbQuery>} filters Resolved. (Should be checked by caller to exist)
- * @param {Array<string>} nodeTypeNames
- * @param {undefined | null | FiltersCache} filtersCache
- * @returns {Array<IGatsbyNode> | undefined} Collection of results
- */
-const filterWithoutSift = (filters, nodeTypeNames, filtersCache) => {
-  if (!filtersCache) {
-    // If no filter cache is passed on, explicitly don't use one
-    return undefined
-  }
-
-  if (filters.length === 0) {
-    // If no filters are given, go through Sift. This does not appear to be
-    // slower than shortcutting it here.
-    return undefined
-  }
-
-  if (
-    filters.some(
-      filter =>
-        filter.type === `query` && !FAST_OPS.includes(filter.query.comparator)
-    )
-  ) {
-    // If there's a filter with non-supported op, stop now.
-    return undefined
-  }
-
-  return runFiltersWithoutSift(filters, nodeTypeNames, filtersCache)
-}
-
-// Not a public API
-exports.filterWithoutSift = filterWithoutSift
-
 /**
  * Use sift to apply filters
  *
  * @param {Array<DbQuery>} filters Resolved
  * @param {boolean} firstOnly
  * @param {Array<string>} nodeTypeNames
  * @param resolvedFields
- * @returns {Array<IGatsbyNode> | undefined | null} Collection of results.
+ * @returns {Array<IGatsbyNode> | null} Collection of results.
  *   Collection will be limited to 1 if `firstOnly` is true
  */
 const filterWithSift = (filters, firstOnly, nodeTypeNames, resolvedFields) => {
@@ -566,7 +535,7 @@ const filterWithSift = (filters, firstOnly, nodeTypeNames, resolvedFields) => {
  * @param {Array<string>} nodeTypeNames
  * @param resolvedFields
  * @param {function(id: string): IGatsbyNode | undefined} getNode
- * @returns {Array<IGatsbyNode> | undefined | null} Collection of results.
+ * @returns {Array<IGatsbyNode> | null} Collection of results.
  *   Collection will be limited to 1 if `firstOnly` is true
  */
 const runSiftOnNodes = (
@@ -605,13 +574,14 @@ const runSiftOnNodes = (
 /**
  * Given a list of filtered nodes and sorting parameters, sort the nodes
  *
- * @param {Array<IGatsbyNode> | undefined | null} nodes Pre-filtered list of nodes
+ * @param {Array<IGatsbyNode> | null} nodes Pre-filtered list of nodes
  * @param {Object | undefined} sort Sorting arguments
  * @param resolvedFields
  * @returns {Array<IGatsbyNode> | undefined | null} Same as input, except sorted
  */
 const sortNodes = (nodes, sort, resolvedFields, stats) => {
-  if (!sort || nodes?.length <= 1) {
+  // `undefined <= 1` and `undefined > 1` are both false so invert the result...
+  if (!sort || !(nodes?.length > 1)) {
     return nodes
   }
 
diff --git a/packages/gatsby/src/schema/__tests__/run-query.js b/packages/gatsby/src/schema/__tests__/run-query.js
@@ -414,6 +414,15 @@ it(`should use the cache argument`, async () => {
 
   describe(desc, () => {
     describe(`Filter fields`, () => {
+      describe(`none`, () => {
+        it(`handles empty filter`, async () => {
+          const [result, allNodes] = await runFastFilter({})
+
+          // Expecting all nodes
+          expect(result?.length).toEqual(allNodes.length)
+        })
+      })
+
       describe(`$eq`, () => {
         it(`handles eq operator with number value`, async () => {
           const needle = 2
