From 2abe9b2f508d6812453394704c4d9915c4ecb2b8 Mon Sep 17 00:00:00 2001 From: Kevin Pruett Date: Mon, 6 Jul 2020 10:05:26 -0400 Subject: [PATCH] Update Algolia CI script --- website/package-lock.json | 254 ++++++++++++++++++------ website/package.json | 4 +- website/scripts/index_search_content.js | 62 ++++-- 3 files changed, 244 insertions(+), 76 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 3f9f104413..0f050864f0 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -1841,6 +1841,119 @@ "unist-util-is": "^4.0.2", "unist-util-map": "^2.0.1", "unist-util-visit": "^2.0.2" + }, + "dependencies": { + "markdown-table": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz", + "integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q==" + }, + "mdast-util-compact": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz", + "integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==", + "requires": { + "unist-util-visit": "^1.1.0" + }, + "dependencies": { + "unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "requires": { + "unist-util-visit-parents": "^2.0.0" + } + } + } + }, + "remark": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz", + "integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==", + "requires": { + "remark-parse": "^7.0.0", + "remark-stringify": "^7.0.0", + "unified": "^8.2.0" + } + }, + "remark-parse": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz", + "integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==", + "requires": { + "collapse-white-space": "^1.0.2", + "is-alphabetical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-whitespace-character": "^1.0.0", + "is-word-character": "^1.0.0", + "markdown-escapes": "^1.0.0", + "parse-entities": "^1.1.0", + "repeat-string": "^1.5.4", + "state-toggle": "^1.0.0", + "trim": "0.0.1", + "trim-trailing-lines": "^1.0.0", + "unherit": "^1.0.4", + "unist-util-remove-position": "^1.0.0", + "vfile-location": "^2.0.0", + "xtend": "^4.0.1" + } + }, + "remark-stringify": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz", + "integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==", + "requires": { + "ccount": "^1.0.0", + "is-alphanumeric": "^1.0.0", + "is-decimal": "^1.0.0", + "is-whitespace-character": "^1.0.0", + "longest-streak": "^2.0.1", + "markdown-escapes": "^1.0.0", + "markdown-table": "^1.1.0", + "mdast-util-compact": "^1.0.0", + "parse-entities": "^1.0.2", + "repeat-string": "^1.5.4", + "state-toggle": "^1.0.0", + "stringify-entities": "^2.0.0", + "unherit": "^1.0.4", + "xtend": "^4.0.1" + } + }, + "stringify-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz", + "integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==", + "requires": { + "character-entities-html4": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.2", + "is-hexadecimal": "^1.0.0" + } + }, + "unist-util-remove-position": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz", + "integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==", + "requires": { + "unist-util-visit": "^1.1.0" + }, + "dependencies": { + "unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "requires": { + "unist-util-visit-parents": "^2.0.0" + } + } + } + }, + "vfile-location": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz", + "integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA==" + } } }, "@mapbox/rehype-prism": { @@ -9370,9 +9483,12 @@ "integrity": "sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg==" }, "markdown-table": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz", - "integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", + "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", + "requires": { + "repeat-string": "^1.0.0" + } }, "marked": { "version": "0.7.0", @@ -9420,21 +9536,11 @@ } }, "mdast-util-compact": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz", - "integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-2.0.1.tgz", + "integrity": "sha512-7GlnT24gEwDrdAwEHrU4Vv5lLWrEer4KOkAiKT9nYstsTad7Oc1TwqT2zIMKRdZF7cTuaf+GA1E4Kv7jJh8mPA==", "requires": { - "unist-util-visit": "^1.1.0" - }, - "dependencies": { - "unist-util-visit": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", - "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", - "requires": { - "unist-util-visit-parents": "^2.0.0" - } - } + "unist-util-visit": "^2.0.0" } }, "mdast-util-definitions": { @@ -12694,13 +12800,28 @@ } }, "remark": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz", - "integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==", + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/remark/-/remark-12.0.0.tgz", + "integrity": "sha512-oX4lMIS0csgk8AEbzY0h2jdR0ngiCHOpwwpxjmRa5TqAkeknY+tkhjRJGZqnCmvyuWh55/0SW5WY3R3nn3PH9A==", "requires": { - "remark-parse": "^7.0.0", - "remark-stringify": "^7.0.0", - "unified": "^8.2.0" + "remark-parse": "^8.0.0", + "remark-stringify": "^8.0.0", + "unified": "^9.0.0" + }, + "dependencies": { + "unified": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/unified/-/unified-9.0.0.tgz", + "integrity": "sha512-ssFo33gljU3PdlWLjNp15Inqb77d6JnJSfyplGJPT/a+fNRNyCBeveBAYJdO5khKdF6WVHa/yYCC7Xl6BDwZUQ==", + "requires": { + "bail": "^1.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^2.0.0", + "trough": "^1.0.0", + "vfile": "^4.0.0" + } + } } }, "remark-footnotes": { @@ -12793,25 +12914,41 @@ } }, "remark-parse": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz", - "integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-8.0.2.tgz", + "integrity": "sha512-eMI6kMRjsAGpMXXBAywJwiwAse+KNpmt+BK55Oofy4KvBZEqUDj6mWbGLJZrujoPIPPxDXzn3T9baRlpsm2jnQ==", "requires": { + "ccount": "^1.0.0", "collapse-white-space": "^1.0.2", "is-alphabetical": "^1.0.0", "is-decimal": "^1.0.0", "is-whitespace-character": "^1.0.0", "is-word-character": "^1.0.0", "markdown-escapes": "^1.0.0", - "parse-entities": "^1.1.0", + "parse-entities": "^2.0.0", "repeat-string": "^1.5.4", "state-toggle": "^1.0.0", "trim": "0.0.1", "trim-trailing-lines": "^1.0.0", "unherit": "^1.0.4", - "unist-util-remove-position": "^1.0.0", - "vfile-location": "^2.0.0", + "unist-util-remove-position": "^2.0.0", + "vfile-location": "^3.0.0", "xtend": "^4.0.1" + }, + "dependencies": { + "parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "requires": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + } + } } }, "remark-squeeze-paragraphs": { @@ -12823,9 +12960,9 @@ } }, "remark-stringify": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz", - "integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-8.1.0.tgz", + "integrity": "sha512-FSPZv1ds76oAZjurhhuV5qXSUSoz6QRPuwYK38S41sLHwg4oB7ejnmZshj7qwjgYLf93kdz6BOX9j5aidNE7rA==", "requires": { "ccount": "^1.0.0", "is-alphanumeric": "^1.0.0", @@ -12833,14 +12970,29 @@ "is-whitespace-character": "^1.0.0", "longest-streak": "^2.0.1", "markdown-escapes": "^1.0.0", - "markdown-table": "^1.1.0", - "mdast-util-compact": "^1.0.0", - "parse-entities": "^1.0.2", + "markdown-table": "^2.0.0", + "mdast-util-compact": "^2.0.0", + "parse-entities": "^2.0.0", "repeat-string": "^1.5.4", "state-toggle": "^1.0.0", - "stringify-entities": "^2.0.0", + "stringify-entities": "^3.0.0", "unherit": "^1.0.4", "xtend": "^4.0.1" + }, + "dependencies": { + "parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "requires": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + } + } } }, "remove-trailing-separator": { @@ -13975,9 +14127,9 @@ } }, "stringify-entities": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz", - "integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-3.0.1.tgz", + "integrity": "sha512-Lsk3ISA2++eJYqBMPKcr/8eby1I6L0gP0NlxF8Zja6c05yr/yCYyb2c9PwXjd08Ib3If1vn1rbs1H5ZtVuOfvQ==", "requires": { "character-entities-html4": "^1.0.0", "character-entities-legacy": "^1.0.0", @@ -15531,21 +15683,11 @@ } }, "unist-util-remove-position": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz", - "integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz", + "integrity": "sha512-fDZsLYIe2uT+oGFnuZmy73K6ZxOPG/Qcm+w7jbEjaFcJgbQ6cqjs/eSPzXhsmGpAsWPkqZM9pYjww5QTn3LHMA==", "requires": { - "unist-util-visit": "^1.1.0" - }, - "dependencies": { - "unist-util-visit": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", - "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", - "requires": { - "unist-util-visit-parents": "^2.0.0" - } - } + "unist-util-visit": "^2.0.0" } }, "unist-util-stringify-position": { @@ -15825,9 +15967,9 @@ } }, "vfile-location": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz", - "integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-3.0.1.tgz", + "integrity": "sha512-yYBO06eeN/Ki6Kh1QAkgzYpWT1d3Qln+ZCtSbJqFExPl1S3y2qqotJQXoh6qEvl/jDlgpUJolBn3PItVnnZRqQ==" }, "vfile-message": { "version": "2.0.4", diff --git a/website/package.json b/website/package.json index 55f70ff8ef..36248236a5 100644 --- a/website/package.json +++ b/website/package.json @@ -45,7 +45,9 @@ "nuka-carousel": "^4.6.7", "react": "^16.13.1", "react-device-detect": "^1.12.1", - "react-dom": "^16.13.1" + "react-dom": "^16.13.1", + "remark": "^12.0.0", + "unist-util-visit": "^2.0.2" }, "devDependencies": { "dart-linkcheck": "^2.0.15", diff --git a/website/scripts/index_search_content.js b/website/scripts/index_search_content.js index 871b542dcf..7eeb7524f1 100644 --- a/website/scripts/index_search_content.js +++ b/website/scripts/index_search_content.js @@ -4,6 +4,8 @@ const algoliasearch = require('algoliasearch') const glob = require('glob') const matter = require('gray-matter') const path = require('path') +const remark = require('remark') +const visit = require('unist-util-visit') // In addition to the content of the page, // define additional front matter attributes that will be search-indexable @@ -15,30 +17,32 @@ async function main() { const pagesFolder = path.join(__dirname, '../pages') // Grab all search-indexable content and format for Algolia - const searchObjects = glob - .sync(path.join(pagesFolder, '**/*.mdx')) - .map((fullPath) => { + const searchObjects = await Promise.all( + glob.sync(path.join(pagesFolder, '**/*.mdx')).map(async (fullPath) => { const { content, data } = matter.read(fullPath) + const searchableDimensions = SEARCH_DIMENSIONS.reduce( + (acc, dimension) => { + return { ...acc, [dimension]: data[dimension] } + }, + {} + ) + + const headings = await collectHeadings(content) + // Get path relative to `pages` const __resourcePath = fullPath.replace(`${pagesFolder}/`, '') // Use clean URL for Algolia id const objectID = __resourcePath.replace('.mdx', '') - const searchableDimensions = Object.keys(data) - .filter((key) => SEARCH_DIMENSIONS.includes(key)) - .map((dimension) => ({ - [dimension]: data[dimension], - })) - return { ...searchableDimensions, - content, - __resourcePath, + headings, objectID, } }) + ) try { await indexSearchContent(searchObjects) @@ -67,25 +71,23 @@ async function indexSearchContent(objects) { const searchClient = algoliasearch(appId, apiKey) const searchIndex = searchClient.initIndex(index) - await searchIndex.partialUpdateObjects(objects, { + const { objectIDs } = await searchIndex.partialUpdateObjects(objects, { createIfNotExists: true, }) - // Remove indices for items that aren't included in the new batch - const newObjectIds = objects.map(({ objectID }) => objectID) - let staleObjects = [] + let staleIds = [] await searchIndex.browseObjects({ query: '', batch: (batch) => { - staleObjects = staleObjects.concat( - batch.filter(({ objectID }) => !newObjectIds.includes(objectID)) + staleIds = staleIds.concat( + batch + .filter(({ objectID }) => !objectIDs.includes(objectID)) + .map(({ objectID }) => objectID) ) }, }) - const staleIds = staleObjects.map(({ objectID }) => objectID) - if (staleIds.length > 0) { console.log(`deleting ${staleIds.length} stale indices:`) console.log(staleIds) @@ -99,3 +101,25 @@ async function indexSearchContent(objects) { throw new Error(error) } } + +async function collectHeadings(mdxContent) { + const headings = [] + + const headingMapper = () => (tree) => { + visit(tree, 'heading', (node) => { + const title = node.children.reduce((m, n) => { + if (n.value) m += n.value + return m + }, '') + // Only include level 1 or level 2 headings + if (node.depth < 3) { + headings.push(title) + } + }) + } + + return remark() + .use(headingMapper) + .process(mdxContent) + .then(() => headings) +}