general non-image file attachments

Part of mdn#3984

build/cli.js

@@ -186,6 +186,7 @@ async function buildDocuments(
       fs.writeFileSync(liveSamplePath, html);
     }
 
+    console.log({ fileAttachments });
     for (const filePath of fileAttachments) {
       // We *could* use symlinks instead. But, there's no point :)
       // Yes, a symlink is less disk I/O but it's nominal.

build/index.js

@@ -10,6 +10,8 @@ const {
   Image,
   REPOSITORY_URLS,
   execGit,
+  VALID_FILE_TYPES,
+  VALID_IMAGE_EXTENSIONS,
 } = require("../content");
 const kumascript = require("../kumascript");
 
@@ -260,18 +262,20 @@ function makeTOC(doc) {
 }
 
 /**
- * Return an array of all images that are inside the documents source folder.
+ * Return an array of all images and files that are inside the documents source folder.
  *
  * @param {Document} document
  */
-function getAdjacentImages(documentDirectory) {
+function getAdjacentFileAttachments(documentDirectory) {
   const dirents = fs.readdirSync(documentDirectory, { withFileTypes: true });
   return dirents
     .filter((dirent) => {
       // This needs to match what we do in filecheck/checker.py
+      const extension = dirent.name.split(".").pop().toLowerCase();
       return (
         !dirent.isDirectory() &&
-        /\.(png|jpeg|jpg|gif|svg|webp)$/i.test(dirent.name)
+        (VALID_IMAGE_EXTENSIONS.has(extension) ||
+          VALID_FILE_TYPES.has(extension))
       );
     })
     .map((dirent) => path.join(documentDirectory, dirent.name));
@@ -449,9 +453,9 @@ async function buildDocument(document, documentOptions = {}) {
   // current document's folder and they might be referenced in live samples.
   // The checkImageReferences() does 2 things. Checks image *references* and
   // it returns which images it checked. But we'll need to complement any
-  // other images in the folder.
-  getAdjacentImages(path.dirname(document.fileInfo.path)).forEach((fp) =>
-    fileAttachments.add(fp)
+  // other images and files in the folder.
+  getAdjacentFileAttachments(path.dirname(document.fileInfo.path)).forEach(
+    (fp) => fileAttachments.add(fp)
   );
 
   // Check the img tags for possible flaws and possible build-time rewrites

client/src/setupProxy.js

@@ -14,4 +14,7 @@ module.exports = function (app) {
   app.use("**/*.(png|webp|gif|jpe?g|svg)", proxy);
   // All those root-level images like /favicon-48x48.png
   app.use("/*.(png|webp|gif|jpe?g|svg)", proxy);
+  // This needs to match what we put into content/constants.js
+  // in the keys of the VALID_FILE_TYPES map.
+  app.use("**/*.(ttf|woff2?)", proxy);
 };

content/constants.js

@@ -55,6 +55,23 @@ if (CONTENT_TRANSLATED_ROOT) {
   REPOSITORY_URLS[CONTENT_TRANSLATED_ROOT] = "mdn/translated-content";
 }
 
+// These are the files we can accept that are not images. The files that can
+// chec checked into the content repo(s).
+const VALID_FILE_TYPES = new Map([
+  // Explanation:
+  // Each row is [fileextension, Array of mime types]
+  ["ttf", ["font/ttf"]],
+]);
+
+const VALID_IMAGE_EXTENSIONS = new Set([
+  "png",
+  "jpeg",
+  "jpg",
+  "gif",
+  "svg",
+  "webp",
+]);
+
 module.exports = {
   CONTENT_ROOT,
   CONTENT_ARCHIVED_ROOT,
@@ -63,4 +80,6 @@ module.exports = {
   ROOTS,
   VALID_LOCALES,
   ACTIVE_LOCALES,
+  VALID_FILE_TYPES,
+  VALID_IMAGE_EXTENSIONS,
 };

content/file.js

@@ -0,0 +1,50 @@
+const fs = require("fs");
+const path = require("path");
+
+const readChunk = require("read-chunk");
+const FileType = require("file-type");
+
+const { ROOTS, VALID_FILE_TYPES } = require("./constants");
+const { memoize, slugToFolder } = require("./utils");
+
+async function isFile(filePath) {
+  if (fs.statSync(filePath).isDirectory()) {
+    return false;
+  }
+
+  const buffer = readChunk.sync(filePath, 0, 12);
+  if (buffer.length === 0) {
+    return false;
+  }
+  const type = await FileType.fromBuffer(buffer);
+  // If the file valid, this comes back
+  // as, for example: { ext: 'ttf', mime: 'font/ttf' }
+  if (!type || !VALID_FILE_TYPES.has(type.ext)) {
+    return false;
+  }
+  if (!VALID_FILE_TYPES.get(type.ext).includes(type.mime)) {
+    console.warn(`${filePath} (${type}) not a valid file type`);
+    return false;
+  }
+
+  return true;
+}
+
+function urlToFilePath(url) {
+  const [, locale, , ...slugParts] = decodeURI(url).split("/");
+  return path.join(locale.toLowerCase(), slugToFolder(slugParts.join("/")));
+}
+
+const find = memoize((relativePath) => {
+  return ROOTS.map((root) => path.join(root, relativePath)).find(
+    (filePath) => fs.existsSync(filePath) && isFile(filePath)
+  );
+});
+
+function findByURL(url) {
+  return find(urlToFilePath(url));
+}
+
+module.exports = {
+  findByURL,
+};

content/image.js

@@ -5,7 +5,7 @@ const readChunk = require("read-chunk");
 const imageType = require("image-type");
 const isSvg = require("is-svg");
 
-const { ROOTS } = require("./constants");
+const { ROOTS, VALID_IMAGE_EXTENSIONS } = require("./constants");
 const { memoize, slugToFolder } = require("./utils");
 
 function isImage(filePath) {
@@ -26,6 +26,9 @@ function isImage(filePath) {
     // https://github.com/sindresorhus/image-type#supported-file-types
     return false;
   }
+  if (!VALID_IMAGE_EXTENSIONS.has(type.ext)) {
+    return false;
+  }
 
   return true;
 }

content/index.js

@@ -5,12 +5,15 @@ const {
   REPOSITORY_URLS,
   ROOTS,
   VALID_LOCALES,
+  VALID_FILE_TYPES,
+  VALID_IMAGE_EXTENSIONS,
 } = require("./constants");
 const Document = require("./document");
 const Translation = require("./translation");
 const { getPopularities } = require("./popularities");
 const Redirect = require("./redirect");
 const Image = require("./image");
+const File = require("./file");
 const Archive = require("./archive");
 const {
   buildURL,
@@ -29,12 +32,15 @@ module.exports = {
   REPOSITORY_URLS,
   ROOTS,
   VALID_LOCALES,
+  VALID_FILE_TYPES,
+  VALID_IMAGE_EXTENSIONS,
 
   getPopularities,
 
   Document,
   Redirect,
   Image,
+  File,
   Archive,
   Translation,
 

server/index.js

@@ -19,6 +19,7 @@ const {
   Document,
   Redirect,
   Image,
+  File,
   CONTENT_TRANSLATED_ROOT,
 } = require("../content");
 // eslint-disable-next-line node/no-missing-require
@@ -211,6 +212,20 @@ app.get("/*", async (req, res) => {
       .sendFile(path.join(STATIC_ROOT, "en-us", "_spas", "404.html"));
   }
 
+  if (/\.(ttf|woff2?)$/.test(req.path)) {
+    const filePath = File.findByURL(req.path);
+    if (filePath) {
+      // The second parameter to `send()` has to be either a full absolute
+      // path or a path that doesn't start with `../` otherwise you'd
+      // get a 403 Forbidden.
+      // See https://github.com/mdn/yari/issues/1297
+      return send(req, path.resolve(filePath)).pipe(res);
+    }
+    // If it's not a file on disk, in the content repo(s), let it try to build
+    // it as a page. For example, there could be a slug whose name is
+    // something "Web/CSS/Fonts/About_the.woff2"
+  }
+
   // TODO: Would be nice to have a list of all supported file extensions
   // in a constants file.
   if (/\.(png|webp|gif|jpe?g|svg)$/.test(req.path)) {