Update Algolia CI script

website/package-lock.json

@@ -1841,6 +1841,119 @@
         "unist-util-is": "^4.0.2",
         "unist-util-map": "^2.0.1",
         "unist-util-visit": "^2.0.2"
+      },
+      "dependencies": {
+        "markdown-table": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz",
+          "integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q=="
+        },
+        "mdast-util-compact": {
+          "version": "1.0.4",
+          "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz",
+          "integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==",
+          "requires": {
+            "unist-util-visit": "^1.1.0"
+          },
+          "dependencies": {
+            "unist-util-visit": {
+              "version": "1.4.1",
+              "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
+              "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
+              "requires": {
+                "unist-util-visit-parents": "^2.0.0"
+              }
+            }
+          }
+        },
+        "remark": {
+          "version": "11.0.2",
+          "resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz",
+          "integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==",
+          "requires": {
+            "remark-parse": "^7.0.0",
+            "remark-stringify": "^7.0.0",
+            "unified": "^8.2.0"
+          }
+        },
+        "remark-parse": {
+          "version": "7.0.2",
+          "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz",
+          "integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==",
+          "requires": {
+            "collapse-white-space": "^1.0.2",
+            "is-alphabetical": "^1.0.0",
+            "is-decimal": "^1.0.0",
+            "is-whitespace-character": "^1.0.0",
+            "is-word-character": "^1.0.0",
+            "markdown-escapes": "^1.0.0",
+            "parse-entities": "^1.1.0",
+            "repeat-string": "^1.5.4",
+            "state-toggle": "^1.0.0",
+            "trim": "0.0.1",
+            "trim-trailing-lines": "^1.0.0",
+            "unherit": "^1.0.4",
+            "unist-util-remove-position": "^1.0.0",
+            "vfile-location": "^2.0.0",
+            "xtend": "^4.0.1"
+          }
+        },
+        "remark-stringify": {
+          "version": "7.0.4",
+          "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz",
+          "integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==",
+          "requires": {
+            "ccount": "^1.0.0",
+            "is-alphanumeric": "^1.0.0",
+            "is-decimal": "^1.0.0",
+            "is-whitespace-character": "^1.0.0",
+            "longest-streak": "^2.0.1",
+            "markdown-escapes": "^1.0.0",
+            "markdown-table": "^1.1.0",
+            "mdast-util-compact": "^1.0.0",
+            "parse-entities": "^1.0.2",
+            "repeat-string": "^1.5.4",
+            "state-toggle": "^1.0.0",
+            "stringify-entities": "^2.0.0",
+            "unherit": "^1.0.4",
+            "xtend": "^4.0.1"
+          }
+        },
+        "stringify-entities": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz",
+          "integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==",
+          "requires": {
+            "character-entities-html4": "^1.0.0",
+            "character-entities-legacy": "^1.0.0",
+            "is-alphanumerical": "^1.0.0",
+            "is-decimal": "^1.0.2",
+            "is-hexadecimal": "^1.0.0"
+          }
+        },
+        "unist-util-remove-position": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz",
+          "integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==",
+          "requires": {
+            "unist-util-visit": "^1.1.0"
+          },
+          "dependencies": {
+            "unist-util-visit": {
+              "version": "1.4.1",
+              "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
+              "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
+              "requires": {
+                "unist-util-visit-parents": "^2.0.0"
+              }
+            }
+          }
+        },
+        "vfile-location": {
+          "version": "2.0.6",
+          "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz",
+          "integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA=="
+        }
       }
     },
     "@mapbox/rehype-prism": {
@@ -9370,9 +9483,12 @@
       "integrity": "sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg=="
     },
     "markdown-table": {
-      "version": "1.1.3",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz",
-      "integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q=="
+      "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==",
+      "requires": {
+        "repeat-string": "^1.0.0"
+      }
     },
     "marked": {
       "version": "0.7.0",
@@ -9420,21 +9536,11 @@
       }
     },
     "mdast-util-compact": {
-      "version": "1.0.4",
+      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz",
+      "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-2.0.1.tgz",
-      "integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==",
+      "integrity": "sha512-7GlnT24gEwDrdAwEHrU4Vv5lLWrEer4KOkAiKT9nYstsTad7Oc1TwqT2zIMKRdZF7cTuaf+GA1E4Kv7jJh8mPA==",
       "requires": {
-        "unist-util-visit": "^1.1.0"
+        "unist-util-visit": "^2.0.0"
-      },
-      "dependencies": {
-        "unist-util-visit": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
-          "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
-          "requires": {
-            "unist-util-visit-parents": "^2.0.0"
-          }
-        }
       }
     },
     "mdast-util-definitions": {
@@ -12694,13 +12800,28 @@
       }
     },
     "remark": {
-      "version": "11.0.2",
+      "version": "12.0.0",
-      "resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/remark/-/remark-12.0.0.tgz",
-      "integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==",
+      "integrity": "sha512-oX4lMIS0csgk8AEbzY0h2jdR0ngiCHOpwwpxjmRa5TqAkeknY+tkhjRJGZqnCmvyuWh55/0SW5WY3R3nn3PH9A==",
       "requires": {
-        "remark-parse": "^7.0.0",
+        "remark-parse": "^8.0.0",
-        "remark-stringify": "^7.0.0",
+        "remark-stringify": "^8.0.0",
-        "unified": "^8.2.0"
+        "unified": "^9.0.0"
+      },
+      "dependencies": {
+        "unified": {
+          "version": "9.0.0",
+          "resolved": "https://registry.npmjs.org/unified/-/unified-9.0.0.tgz",
+          "integrity": "sha512-ssFo33gljU3PdlWLjNp15Inqb77d6JnJSfyplGJPT/a+fNRNyCBeveBAYJdO5khKdF6WVHa/yYCC7Xl6BDwZUQ==",
+          "requires": {
+            "bail": "^1.0.0",
+            "extend": "^3.0.0",
+            "is-buffer": "^2.0.0",
+            "is-plain-obj": "^2.0.0",
+            "trough": "^1.0.0",
+            "vfile": "^4.0.0"
+          }
+        }
       }
     },
     "remark-footnotes": {
@@ -12793,25 +12914,41 @@
       }
     },
     "remark-parse": {
-      "version": "7.0.2",
+      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-8.0.2.tgz",
-      "integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==",
+      "integrity": "sha512-eMI6kMRjsAGpMXXBAywJwiwAse+KNpmt+BK55Oofy4KvBZEqUDj6mWbGLJZrujoPIPPxDXzn3T9baRlpsm2jnQ==",
       "requires": {
+        "ccount": "^1.0.0",
         "collapse-white-space": "^1.0.2",
         "is-alphabetical": "^1.0.0",
         "is-decimal": "^1.0.0",
         "is-whitespace-character": "^1.0.0",
         "is-word-character": "^1.0.0",
         "markdown-escapes": "^1.0.0",
-        "parse-entities": "^1.1.0",
+        "parse-entities": "^2.0.0",
         "repeat-string": "^1.5.4",
         "state-toggle": "^1.0.0",
         "trim": "0.0.1",
         "trim-trailing-lines": "^1.0.0",
         "unherit": "^1.0.4",
-        "unist-util-remove-position": "^1.0.0",
+        "unist-util-remove-position": "^2.0.0",
-        "vfile-location": "^2.0.0",
+        "vfile-location": "^3.0.0",
         "xtend": "^4.0.1"
+      },
+      "dependencies": {
+        "parse-entities": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz",
+          "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==",
+          "requires": {
+            "character-entities": "^1.0.0",
+            "character-entities-legacy": "^1.0.0",
+            "character-reference-invalid": "^1.0.0",
+            "is-alphanumerical": "^1.0.0",
+            "is-decimal": "^1.0.0",
+            "is-hexadecimal": "^1.0.0"
+          }
+        }
       }
     },
     "remark-squeeze-paragraphs": {
@@ -12823,9 +12960,9 @@
       }
     },
     "remark-stringify": {
-      "version": "7.0.4",
+      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz",
+      "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-8.1.0.tgz",
-      "integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==",
+      "integrity": "sha512-FSPZv1ds76oAZjurhhuV5qXSUSoz6QRPuwYK38S41sLHwg4oB7ejnmZshj7qwjgYLf93kdz6BOX9j5aidNE7rA==",
       "requires": {
         "ccount": "^1.0.0",
         "is-alphanumeric": "^1.0.0",
@@ -12833,14 +12970,29 @@
         "is-whitespace-character": "^1.0.0",
         "longest-streak": "^2.0.1",
         "markdown-escapes": "^1.0.0",
-        "markdown-table": "^1.1.0",
+        "markdown-table": "^2.0.0",
-        "mdast-util-compact": "^1.0.0",
+        "mdast-util-compact": "^2.0.0",
-        "parse-entities": "^1.0.2",
+        "parse-entities": "^2.0.0",
         "repeat-string": "^1.5.4",
         "state-toggle": "^1.0.0",
-        "stringify-entities": "^2.0.0",
+        "stringify-entities": "^3.0.0",
         "unherit": "^1.0.4",
         "xtend": "^4.0.1"
+      },
+      "dependencies": {
+        "parse-entities": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz",
+          "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==",
+          "requires": {
+            "character-entities": "^1.0.0",
+            "character-entities-legacy": "^1.0.0",
+            "character-reference-invalid": "^1.0.0",
+            "is-alphanumerical": "^1.0.0",
+            "is-decimal": "^1.0.0",
+            "is-hexadecimal": "^1.0.0"
+          }
+        }
       }
     },
     "remove-trailing-separator": {
@@ -13975,9 +14127,9 @@
       }
     },
     "stringify-entities": {
-      "version": "2.0.0",
+      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-3.0.1.tgz",
-      "integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==",
+      "integrity": "sha512-Lsk3ISA2++eJYqBMPKcr/8eby1I6L0gP0NlxF8Zja6c05yr/yCYyb2c9PwXjd08Ib3If1vn1rbs1H5ZtVuOfvQ==",
       "requires": {
         "character-entities-html4": "^1.0.0",
         "character-entities-legacy": "^1.0.0",
@@ -15531,21 +15683,11 @@
       }
     },
     "unist-util-remove-position": {
-      "version": "1.1.4",
+      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz",
+      "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz",
-      "integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==",
+      "integrity": "sha512-fDZsLYIe2uT+oGFnuZmy73K6ZxOPG/Qcm+w7jbEjaFcJgbQ6cqjs/eSPzXhsmGpAsWPkqZM9pYjww5QTn3LHMA==",
       "requires": {
-        "unist-util-visit": "^1.1.0"
+        "unist-util-visit": "^2.0.0"
-      },
-      "dependencies": {
-        "unist-util-visit": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
-          "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
-          "requires": {
-            "unist-util-visit-parents": "^2.0.0"
-          }
-        }
       }
     },
     "unist-util-stringify-position": {
@@ -15825,9 +15967,9 @@
       }
     },
     "vfile-location": {
-      "version": "2.0.6",
+      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz",
+      "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-3.0.1.tgz",
-      "integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA=="
+      "integrity": "sha512-yYBO06eeN/Ki6Kh1QAkgzYpWT1d3Qln+ZCtSbJqFExPl1S3y2qqotJQXoh6qEvl/jDlgpUJolBn3PItVnnZRqQ=="
     },
     "vfile-message": {
       "version": "2.0.4",

website/package.json

@@ -45,7 +45,9 @@
     "nuka-carousel": "^4.6.7",
     "react": "^16.13.1",
     "react-device-detect": "^1.12.1",
-    "react-dom": "^16.13.1"
+    "react-dom": "^16.13.1",
+    "remark": "^12.0.0",
+    "unist-util-visit": "^2.0.2"
   },
   "devDependencies": {
     "dart-linkcheck": "^2.0.15",

website/scripts/index_search_content.js

@@ -4,6 +4,8 @@ const algoliasearch = require('algoliasearch')
 const glob = require('glob')
 const matter = require('gray-matter')
 const path = require('path')
+const remark = require('remark')
+const visit = require('unist-util-visit')
 
 // In addition to the content of the page,
 // define additional front matter attributes that will be search-indexable
@@ -15,30 +17,32 @@ async function main() {
   const pagesFolder = path.join(__dirname, '../pages')
 
   // Grab all search-indexable content and format for Algolia
-  const searchObjects = glob
+  const searchObjects = await Promise.all(
-    .sync(path.join(pagesFolder, '**/*.mdx'))
+    glob.sync(path.join(pagesFolder, '**/*.mdx')).map(async (fullPath) => {
-    .map((fullPath) => {
       const { content, data } = matter.read(fullPath)
 
+      const searchableDimensions = SEARCH_DIMENSIONS.reduce(
+        (acc, dimension) => {
+          return { ...acc, [dimension]: data[dimension] }
+        },
+        {}
+      )
+
+      const headings = await collectHeadings(content)
+
       // Get path relative to `pages`
       const __resourcePath = fullPath.replace(`${pagesFolder}/`, '')
 
       // Use clean URL for Algolia id
       const objectID = __resourcePath.replace('.mdx', '')
 
-      const searchableDimensions = Object.keys(data)
-        .filter((key) => SEARCH_DIMENSIONS.includes(key))
-        .map((dimension) => ({
-          [dimension]: data[dimension],
-        }))
-
       return {
         ...searchableDimensions,
-        content,
+        headings,
-        __resourcePath,
         objectID,
       }
     })
+  )
 
   try {
     await indexSearchContent(searchObjects)
@@ -67,25 +71,23 @@ async function indexSearchContent(objects) {
     const searchClient = algoliasearch(appId, apiKey)
     const searchIndex = searchClient.initIndex(index)
 
-    await searchIndex.partialUpdateObjects(objects, {
+    const { objectIDs } = await searchIndex.partialUpdateObjects(objects, {
       createIfNotExists: true,
     })
 
-    // Remove indices for items that aren't included in the new batch
+    let staleIds = []
-    const newObjectIds = objects.map(({ objectID }) => objectID)
-    let staleObjects = []
 
     await searchIndex.browseObjects({
       query: '',
       batch: (batch) => {
-        staleObjects = staleObjects.concat(
+        staleIds = staleIds.concat(
-          batch.filter(({ objectID }) => !newObjectIds.includes(objectID))
+          batch
+            .filter(({ objectID }) => !objectIDs.includes(objectID))
+            .map(({ objectID }) => objectID)
         )
       },
     })
 
-    const staleIds = staleObjects.map(({ objectID }) => objectID)
-
     if (staleIds.length > 0) {
       console.log(`deleting ${staleIds.length} stale indices:`)
       console.log(staleIds)
@@ -99,3 +101,25 @@ async function indexSearchContent(objects) {
     throw new Error(error)
   }
 }
+
+async function collectHeadings(mdxContent) {
+  const headings = []
+
+  const headingMapper = () => (tree) => {
+    visit(tree, 'heading', (node) => {
+      const title = node.children.reduce((m, n) => {
+        if (n.value) m += n.value
+        return m
+      }, '')
+      // Only include level 1 or level 2 headings
+      if (node.depth < 3) {
+        headings.push(title)
+      }
+    })
+  }
+
+  return remark()
+    .use(headingMapper)
+    .process(mdxContent)
+    .then(() => headings)
+}