|
| 1 | +/** |
| 2 | + * @fileoverview Script to fetch link data. |
| 3 | + * |
| 4 | + * To fetch info about all files: |
| 5 | + * |
| 6 | + * node tools/fetch-docs-links.js |
| 7 | + * |
| 8 | + * To fetch info for just selected files (for use with lint-staged): |
| 9 | + * |
| 10 | + * node tools/fetch-docs-links.js docs/src/user-guide/index.md |
| 11 | + * |
| 12 | + * @author Nicholas C. Zakas |
| 13 | + */ |
| 14 | + |
| 15 | +"use strict"; |
| 16 | + |
| 17 | +//----------------------------------------------------------------------------- |
| 18 | +// Requirements |
| 19 | +//----------------------------------------------------------------------------- |
| 20 | + |
| 21 | +const matter = require("gray-matter"); |
| 22 | +const metascraper = require("metascraper")([ |
| 23 | + require("metascraper-image")(), |
| 24 | + require("metascraper-logo")(), |
| 25 | + require("metascraper-logo-favicon")(), |
| 26 | + require("metascraper-title")(), |
| 27 | + require("metascraper-description")() |
| 28 | +]); |
| 29 | +const got = require("got"); |
| 30 | +const path = require("path"); |
| 31 | +const fs = require("fs/promises"); |
| 32 | +const glob = require("fast-glob"); |
| 33 | + |
| 34 | +//----------------------------------------------------------------------------- |
| 35 | +// Data |
| 36 | +//----------------------------------------------------------------------------- |
| 37 | + |
| 38 | +const BASE_DIR = path.resolve(__dirname, "../"); |
| 39 | +const SRC_DIR = path.resolve(BASE_DIR, "docs/src"); |
| 40 | +const DATA_DIR = path.resolve(SRC_DIR, "_data"); |
| 41 | +const DATA_FILE_PATH = path.resolve(DATA_DIR, "further_reading_links.json"); |
| 42 | + |
| 43 | +// determine which files to check |
| 44 | +let filenames = process.argv.slice(2); |
| 45 | + |
| 46 | +if (filenames.length === 0) { |
| 47 | + filenames = glob.sync("docs/src/rules/*.md", { cwd: BASE_DIR }); |
| 48 | +} |
| 49 | + |
| 50 | +filenames = filenames.map(filename => path.resolve(BASE_DIR, filename)); |
| 51 | + |
| 52 | +//----------------------------------------------------------------------------- |
| 53 | +// Helpers |
| 54 | +//----------------------------------------------------------------------------- |
| 55 | + |
| 56 | +/** |
| 57 | + * Fetches metadata information for a given URL. |
| 58 | + * @param {string} url The URL to fetch data for. |
| 59 | + * @returns {Promise<object>} An object with metadata info. |
| 60 | + */ |
| 61 | +async function fetchLinkMeta(url) { |
| 62 | + const { body: html, url: returnedURL } = await got(url); |
| 63 | + const metadata = await metascraper({ html, url: returnedURL }); |
| 64 | + const domain = (new URL(returnedURL)).hostname; |
| 65 | + |
| 66 | + return { |
| 67 | + domain, |
| 68 | + url, |
| 69 | + logo: metadata.logo, |
| 70 | + title: metadata.title, |
| 71 | + description: metadata.description |
| 72 | + }; |
| 73 | +} |
| 74 | + |
| 75 | + |
| 76 | +//----------------------------------------------------------------------------- |
| 77 | +// Main |
| 78 | +//----------------------------------------------------------------------------- |
| 79 | + |
| 80 | +(async () => { |
| 81 | + |
| 82 | + // First read in the current data file |
| 83 | + const links = JSON.parse(await fs.readFile(DATA_FILE_PATH, "utf8")); |
| 84 | + |
| 85 | + // check each file |
| 86 | + for (const filename of filenames) { |
| 87 | + |
| 88 | + const text = await fs.readFile(filename, "utf8"); |
| 89 | + const frontmatter = matter(text).data; |
| 90 | + |
| 91 | + if (frontmatter.further_reading) { |
| 92 | + |
| 93 | + for (const url of frontmatter.further_reading) { |
| 94 | + if (!links[url]) { |
| 95 | + try { |
| 96 | + links[url] = await fetchLinkMeta(url); |
| 97 | + } catch (ex) { |
| 98 | + console.error("Error in ", filename); |
| 99 | + console.error("Could not fetch data for", url); |
| 100 | + console.error(ex.message); |
| 101 | + console.error(ex.stack); |
| 102 | + process.exit(1); |
| 103 | + } |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + } |
| 108 | + } |
| 109 | + |
| 110 | + // Last write new data into the current data file |
| 111 | + await fs.writeFile(DATA_FILE_PATH, JSON.stringify(links, null, 4), "utf8"); |
| 112 | +})(); |
0 commit comments