github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #8242 from hashicorp/pruett.ci-fixup

Browse Source 
Update Algolia CI script
pull/8244/head
Kevin Pruett 4 years ago committed by GitHub
parent a7f69b615a 2abe9b2f50
commit 52d05480c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 244 additions and 76 deletions
Show Stats Download Patch File Download Diff File

254
website/package-lock.json generated
Unescape View File

@ -3733,6 +3733,119 @@
"unist-util-is": "^4.0.2",
"unist-util-map": "^2.0.1",
"unist-util-visit": "^2.0.2"
},
"dependencies": {
"markdown-table": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz",
"integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q=="
},
"mdast-util-compact": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz",
"integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==",
"requires": {
"unist-util-visit": "^1.1.0"
},
"dependencies": {
"unist-util-visit": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
"integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
"requires": {
"unist-util-visit-parents": "^2.0.0"
}
}
}
},
"remark": {
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz",
"integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==",
"requires": {
"remark-parse": "^7.0.0",
"remark-stringify": "^7.0.0",
"unified": "^8.2.0"
}
},
"remark-parse": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz",
"integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==",
"requires": {
"collapse-white-space": "^1.0.2",
"is-alphabetical": "^1.0.0",
"is-decimal": "^1.0.0",
"is-whitespace-character": "^1.0.0",
"is-word-character": "^1.0.0",
"markdown-escapes": "^1.0.0",
"parse-entities": "^1.1.0",
"repeat-string": "^1.5.4",
"state-toggle": "^1.0.0",
"trim": "0.0.1",
"trim-trailing-lines": "^1.0.0",
"unherit": "^1.0.4",
"unist-util-remove-position": "^1.0.0",
"vfile-location": "^2.0.0",
"xtend": "^4.0.1"
}
},
"remark-stringify": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz",
"integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==",
"requires": {
"ccount": "^1.0.0",
"is-alphanumeric": "^1.0.0",
"is-decimal": "^1.0.0",
"is-whitespace-character": "^1.0.0",
"longest-streak": "^2.0.1",
"markdown-escapes": "^1.0.0",
"markdown-table": "^1.1.0",
"mdast-util-compact": "^1.0.0",
"parse-entities": "^1.0.2",
"repeat-string": "^1.5.4",
"state-toggle": "^1.0.0",
"stringify-entities": "^2.0.0",
"unherit": "^1.0.4",
"xtend": "^4.0.1"
}
},
"stringify-entities": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz",
"integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==",
"requires": {
"character-entities-html4": "^1.0.0",
"character-entities-legacy": "^1.0.0",
"is-alphanumerical": "^1.0.0",
"is-decimal": "^1.0.2",
"is-hexadecimal": "^1.0.0"
}
},
"unist-util-remove-position": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz",
"integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==",
"requires": {
"unist-util-visit": "^1.1.0"
},
"dependencies": {
"unist-util-visit": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
"integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
"requires": {
"unist-util-visit-parents": "^2.0.0"
}
}
}
},
"vfile-location": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz",
"integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA=="
}
}
},
"@mapbox/rehype-prism": {
@ -11170,9 +11283,12 @@
"integrity": "sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg=="
},
"markdown-table": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.3.tgz",
"integrity": "sha512-1RUZVgQlpJSPWYbFSpmudq5nHY1doEIv89gBtF0s4gW1GF2XorxcA/70M5vq7rLv0a6mhOUccRsqkwhwLCIQ2Q=="
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz",
"integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==",
"requires": {
"repeat-string": "^1.0.0"
}
},
"marked": {
"version": "0.7.0",
@ -11220,21 +11336,11 @@
}
},
"mdast-util-compact": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.4.tgz",
"integrity": "sha512-3YDMQHI5vRiS2uygEFYaqckibpJtKq5Sj2c8JioeOQBU6INpKbdWzfyLqFFnDwEcEnRFIdMsguzs5pC1Jp4Isg==",
"requires": {
"unist-util-visit": "^1.1.0"
},
"dependencies": {
"unist-util-visit": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
"integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-2.0.1.tgz",
"integrity": "sha512-7GlnT24gEwDrdAwEHrU4Vv5lLWrEer4KOkAiKT9nYstsTad7Oc1TwqT2zIMKRdZF7cTuaf+GA1E4Kv7jJh8mPA==",
"requires": {
"unist-util-visit-parents": "^2.0.0"
}
}
"unist-util-visit": "^2.0.0"
}
},
"mdast-util-definitions": {
@ -14490,13 +14596,28 @@
}
},
"remark": {
"version": "11.0.2",
"resolved": "https://registry.npmjs.org/remark/-/remark-11.0.2.tgz",
"integrity": "sha512-bh+eJgn8wgmbHmIBOuwJFdTVRVpl3fcVP6HxmpPWO0ULGP9Qkh6INJh0N5Uy7GqlV7DQYGoqaKiEIpM5LLvJ8w==",
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/remark/-/remark-12.0.0.tgz",
"integrity": "sha512-oX4lMIS0csgk8AEbzY0h2jdR0ngiCHOpwwpxjmRa5TqAkeknY+tkhjRJGZqnCmvyuWh55/0SW5WY3R3nn3PH9A==",
"requires": {
"remark-parse": "^7.0.0",
"remark-stringify": "^7.0.0",
"unified": "^8.2.0"
"remark-parse": "^8.0.0",
"remark-stringify": "^8.0.0",
"unified": "^9.0.0"
},
"dependencies": {
"unified": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/unified/-/unified-9.0.0.tgz",
"integrity": "sha512-ssFo33gljU3PdlWLjNp15Inqb77d6JnJSfyplGJPT/a+fNRNyCBeveBAYJdO5khKdF6WVHa/yYCC7Xl6BDwZUQ==",
"requires": {
"bail": "^1.0.0",
"extend": "^3.0.0",
"is-buffer": "^2.0.0",
"is-plain-obj": "^2.0.0",
"trough": "^1.0.0",
"vfile": "^4.0.0"
}
}
}
},
"remark-footnotes": {
@ -14589,25 +14710,41 @@
}
},
"remark-parse": {
"version": "7.0.2",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-7.0.2.tgz",
"integrity": "sha512-9+my0lQS80IQkYXsMA8Sg6m9QfXYJBnXjWYN5U+kFc5/n69t+XZVXU/ZBYr3cYH8FheEGf1v87rkFDhJ8bVgMA==",
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-8.0.2.tgz",
"integrity": "sha512-eMI6kMRjsAGpMXXBAywJwiwAse+KNpmt+BK55Oofy4KvBZEqUDj6mWbGLJZrujoPIPPxDXzn3T9baRlpsm2jnQ==",
"requires": {
"ccount": "^1.0.0",
"collapse-white-space": "^1.0.2",
"is-alphabetical": "^1.0.0",
"is-decimal": "^1.0.0",
"is-whitespace-character": "^1.0.0",
"is-word-character": "^1.0.0",
"markdown-escapes": "^1.0.0",
"parse-entities": "^1.1.0",
"parse-entities": "^2.0.0",
"repeat-string": "^1.5.4",
"state-toggle": "^1.0.0",
"trim": "0.0.1",
"trim-trailing-lines": "^1.0.0",
"unherit": "^1.0.4",
"unist-util-remove-position": "^1.0.0",
"vfile-location": "^2.0.0",
"unist-util-remove-position": "^2.0.0",
"vfile-location": "^3.0.0",
"xtend": "^4.0.1"
},
"dependencies": {
"parse-entities": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz",
"integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==",
"requires": {
"character-entities": "^1.0.0",
"character-entities-legacy": "^1.0.0",
"character-reference-invalid": "^1.0.0",
"is-alphanumerical": "^1.0.0",
"is-decimal": "^1.0.0",
"is-hexadecimal": "^1.0.0"
}
}
}
},
"remark-squeeze-paragraphs": {
@ -14619,9 +14756,9 @@
}
},
"remark-stringify": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-7.0.4.tgz",
"integrity": "sha512-qck+8NeA1D0utk1ttKcWAoHRrJxERYQzkHDyn+pF5Z4whX1ug98uCNPPSeFgLSaNERRxnD6oxIug6DzZQth6Pg==",
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-8.1.0.tgz",
"integrity": "sha512-FSPZv1ds76oAZjurhhuV5qXSUSoz6QRPuwYK38S41sLHwg4oB7ejnmZshj7qwjgYLf93kdz6BOX9j5aidNE7rA==",
"requires": {
"ccount": "^1.0.0",
"is-alphanumeric": "^1.0.0",
@ -14629,14 +14766,29 @@
"is-whitespace-character": "^1.0.0",
"longest-streak": "^2.0.1",
"markdown-escapes": "^1.0.0",
"markdown-table": "^1.1.0",
"mdast-util-compact": "^1.0.0",
"parse-entities": "^1.0.2",
"markdown-table": "^2.0.0",
"mdast-util-compact": "^2.0.0",
"parse-entities": "^2.0.0",
"repeat-string": "^1.5.4",
"state-toggle": "^1.0.0",
"stringify-entities": "^2.0.0",
"stringify-entities": "^3.0.0",
"unherit": "^1.0.4",
"xtend": "^4.0.1"
},
"dependencies": {
"parse-entities": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz",
"integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==",
"requires": {
"character-entities": "^1.0.0",
"character-entities-legacy": "^1.0.0",
"character-reference-invalid": "^1.0.0",
"is-alphanumerical": "^1.0.0",
"is-decimal": "^1.0.0",
"is-hexadecimal": "^1.0.0"
}
}
}
},
"remove-trailing-separator": {
@ -15775,9 +15927,9 @@
}
},
"stringify-entities": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-2.0.0.tgz",
"integrity": "sha512-fqqhZzXyAM6pGD9lky/GOPq6V4X0SeTAFBl0iXb/BzOegl40gpf/bV3QQP7zULNYvjr6+Dx8SCaDULjVoOru0A==",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-3.0.1.tgz",
"integrity": "sha512-Lsk3ISA2++eJYqBMPKcr/8eby1I6L0gP0NlxF8Zja6c05yr/yCYyb2c9PwXjd08Ib3If1vn1rbs1H5ZtVuOfvQ==",
"requires": {
"character-entities-html4": "^1.0.0",
"character-entities-legacy": "^1.0.0",
@ -17332,21 +17484,11 @@
}
},
"unist-util-remove-position": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.4.tgz",
"integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==",
"requires": {
"unist-util-visit": "^1.1.0"
},
"dependencies": {
"unist-util-visit": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz",
"integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz",
"integrity": "sha512-fDZsLYIe2uT+oGFnuZmy73K6ZxOPG/Qcm+w7jbEjaFcJgbQ6cqjs/eSPzXhsmGpAsWPkqZM9pYjww5QTn3LHMA==",
"requires": {
"unist-util-visit-parents": "^2.0.0"
}
}
"unist-util-visit": "^2.0.0"
}
},
"unist-util-stringify-position": {
@ -17626,9 +17768,9 @@
}
},
"vfile-location": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.6.tgz",
"integrity": "sha512-sSFdyCP3G6Ka0CEmN83A2YCMKIieHx0EDaj5IDP4g1pa5ZJ4FJDvpO0WODLxo4LUX4oe52gmSCK7Jw4SBghqxA=="
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-3.0.1.tgz",
"integrity": "sha512-yYBO06eeN/Ki6Kh1QAkgzYpWT1d3Qln+ZCtSbJqFExPl1S3y2qqotJQXoh6qEvl/jDlgpUJolBn3PItVnnZRqQ=="
},
"vfile-message": {
"version": "2.0.4",

4
website/package.json
Unescape View File

@ -43,7 +43,9 @@
"nuka-carousel": "^4.7.0",
"react": "^16.13.1",
"react-device-detect": "^1.12.1",
"react-dom": "^16.13.1"
"react-dom": "^16.13.1",
"remark": "^12.0.0",
"unist-util-visit": "^2.0.2"
},
"devDependencies": {
"dart-linkcheck": "^2.0.15",

62
website/scripts/index_search_content.js
Unescape View File

@ -4,6 +4,8 @@ const algoliasearch = require('algoliasearch')
const glob = require('glob')
const matter = require('gray-matter')
const path = require('path')
const remark = require('remark')
const visit = require('unist-util-visit')
// In addition to the content of the page,
// define additional front matter attributes that will be search-indexable
@ -15,30 +17,32 @@ async function main() {
const pagesFolder = path.join(__dirname, '../pages')
// Grab all search-indexable content and format for Algolia
const searchObjects = glob
.sync(path.join(pagesFolder, '**/*.mdx'))
.map((fullPath) => {
const searchObjects = await Promise.all(
glob.sync(path.join(pagesFolder, '**/*.mdx')).map(async (fullPath) => {
const { content, data } = matter.read(fullPath)
const searchableDimensions = SEARCH_DIMENSIONS.reduce(
(acc, dimension) => {
return { ...acc, [dimension]: data[dimension] }
},
{}
)
const headings = await collectHeadings(content)
// Get path relative to `pages`
const __resourcePath = fullPath.replace(`${pagesFolder}/`, '')
// Use clean URL for Algolia id
const objectID = __resourcePath.replace('.mdx', '')
const searchableDimensions = Object.keys(data)
.filter((key) => SEARCH_DIMENSIONS.includes(key))
.map((dimension) => ({
[dimension]: data[dimension],
}))
return {
...searchableDimensions,
content,
__resourcePath,
headings,
objectID,
}
})
)
try {
await indexSearchContent(searchObjects)
@ -67,25 +71,23 @@ async function indexSearchContent(objects) {
const searchClient = algoliasearch(appId, apiKey)
const searchIndex = searchClient.initIndex(index)
await searchIndex.partialUpdateObjects(objects, {
const { objectIDs } = await searchIndex.partialUpdateObjects(objects, {
createIfNotExists: true,
})
// Remove indices for items that aren't included in the new batch
const newObjectIds = objects.map(({ objectID }) => objectID)
let staleObjects = []
let staleIds = []
await searchIndex.browseObjects({
query: '',
batch: (batch) => {
staleObjects = staleObjects.concat(
batch.filter(({ objectID }) => !newObjectIds.includes(objectID))
staleIds = staleIds.concat(
batch
.filter(({ objectID }) => !objectIDs.includes(objectID))
.map(({ objectID }) => objectID)
)
},
})
const staleIds = staleObjects.map(({ objectID }) => objectID)
if (staleIds.length > 0) {
console.log(`deleting ${staleIds.length} stale indices:`)
console.log(staleIds)
@ -99,3 +101,25 @@ async function indexSearchContent(objects) {
throw new Error(error)
}
}
async function collectHeadings(mdxContent) {
const headings = []
const headingMapper = () => (tree) => {
visit(tree, 'heading', (node) => {
const title = node.children.reduce((m, n) => {
if (n.value) m += n.value
return m
}, '')
// Only include level 1 or level 2 headings
if (node.depth < 3) {
headings.push(title)
}
})
}
return remark()
.use(headingMapper)
.process(mdxContent)
.then(() => headings)
}

Write Preview
Loading…
Cancel
Save