simulate content-origin-request (#3047)

* simulate content-origin-request

* specifics

* oops

* eslint

* start server in background

* polka is smaller

* calm down eslint

* debugging

* node_modules caching

* undebugging

* add basic unit test start

* eslint

* querystring

* more tests

* more tests and stuff

* tests updated based on feedback

* way more conservative with caching node_modules

* all feedback addressed

* Update .github/workflows/content-origin-request.yml

Co-authored-by: Ryan Johnson <escattone@gmail.com>

Co-authored-by: Ryan Johnson <escattone@gmail.com>

.github/workflows/content-origin-request.yml

@@ -0,0 +1,62 @@
+# This starts up a simulator that tries to do what our Lambda@Edge does.
+
+name: content-origin-request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - deployer/aws-lambda/**
+      - libs/**
+      - .github/workflows/content-origin-request.yml
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v2.1.5
+        with:
+          node-version: "12"
+
+      - name: Cache node_modules
+        uses: actions/cache@v2.1.4
+        id: cached-node_modules
+        with:
+          path: |
+            deployer/aws-lambda/content-origin-request/node_modules
+          key: ${{ runner.os }}-${{ hashFiles('deployer/aws-lambda/content-origin-request/yarn.lock') }}-${{ hashFiles('libs/**/*.js') }}
+
+      - name: Install all yarn packages
+        if: steps.cached-node_modules.outputs.cache-hit != 'true'
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: yarn --frozen-lockfile
+
+      - name: Run test server
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: |
+          yarn serve > /tmp/stdout.log 2> /tmp/stderr.log &
+
+      - name: Check that the server started
+        run: curl --retry-connrefused --retry 5 -I http://localhost:7000/ping
+
+      - name: Preflight the integration tests
+        run: |
+          curl -I http://localhost:7000/docs/Web
+          curl -I http://localhost:7000/en-US/docs/Web/
+
+      - name: Unit test
+        working-directory: deployer/aws-lambda/content-origin-request
+        run: |
+          yarn test-server
+
+      - name: Debug any server outputs
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log

deployer/aws-lambda/README.md

@@ -54,3 +54,48 @@ yarn make-package
 
 and if the deployment package is different from what is already in AWS,
 it will upload and publish a new version.
+
+## Debugging the `content-origin-request` handler
+
+You can simulate what Lambda@Edge does, but on your laptop.
+To start it, run:
+
+```sh
+cd aws-lambda
+cd content-origin-request
+yarn install
+yarn serve
+```
+
+This will start a server at <http://localhost:7000>. It's meant to work much
+the same as when our Lambda@Edge function is run within AWS. To test it, try:
+
+```sh
+curl -I http://localhost:7000/EN-us/docs/Foo/
+```
+
+It doesn't actually look things up on disk like CloudFront + Lambda@Edge can do.
+But it's a great tool for end-to-end testing our redirect rules.
+
+### An important caveat about `@yari-internals`
+
+The `yarn serve` server will automatically restart itself if a change is
+made to the `index.js` or the `server.js` code.
+But, if you make an edit to any of the `/libs/**/index.js` files (they're called
+`@yari-internal/...` from within the code), then the only way to get them
+to become the latest version is to run:
+
+```sh
+yarn install --force
+```
+
+This is necessary because they're not versioned.
+
+### Headers
+
+All the headers that the Express server receives it replicates. This means you can
+test things like this:
+
+```sh
+curl -I -H 'accept-language: fr' -H 'cookie: preferredlocale=de' http://localhost:7000/docs/Web
+```

deployer/aws-lambda/content-origin-request/index.js

@@ -132,7 +132,6 @@ exports.handler = async (event) => {
     // processed by this Lambda function, note that we'll remove
     // the trailing slash before the request reaches S3 (see below).
     return redirect(request.uri + "/" + qs, {
-      status: 301,
       cacheControlSeconds: THIRTY_DAYS,
     });
   } else if (LOCALE_URI_WITH_TRAILING_SLASH.has(requestURILowerCase)) {
@@ -151,7 +150,6 @@ exports.handler = async (event) => {
     // All other requests with a trailing slash should redirect to the
     // same URL without the trailing slash.
     return redirect(request.uri.slice(0, -1) + qs, {
-      status: 301,
       cacheControlSeconds: THIRTY_DAYS,
     });
   }

deployer/aws-lambda/content-origin-request/package.json

@@ -4,7 +4,9 @@
   "license": "MPL-2.0",
   "main": "index.js",
   "scripts": {
-    "make-package": "yarn install && zip -r -X function.zip . -i index.js 'node_modules/*'"
+    "make-package": "yarn install --production && zip -r -X function.zip . -i index.js 'node_modules/*'",
+    "serve": "nodemon server.js",
+    "test-server": "jest"
   },
   "dependencies": {
     "@yari-internal/constants": "file:../../../libs/constants",
@@ -15,6 +17,13 @@
     "cookie": "0.4.1",
     "sanitize-filename": "^1.6.3"
   },
+  "devDependencies": {
+    "got": "11.8.1",
+    "jest": "26.6.3",
+    "kleur": "4.1.4",
+    "nodemon": "2.0.7",
+    "polka": "0.5.2"
+  },
   "engines": {
     "node": ">=12.x"
   },

deployer/aws-lambda/content-origin-request/server.js

@@ -0,0 +1,101 @@
+/* eslint-disable node/no-unpublished-require */
+/* eslint-disable node/no-missing-require */
+const polka = require("polka");
+const kleur = require("kleur");
+
+const { handler } = require("./index");
+
+const PORT = parseInt(process.env.PORT || "7000");
+
+function ping(req, res) {
+  res.end("pong\n");
+}
+
+async function catchall(req, res) {
+  // Reminder...
+  // - url: e.g. `/en-US/docs/Foo?key=value`
+  // - search: e.g. `?key=value`
+  const { url } = req;
+  let querystring = "";
+  if (req.search) {
+    querystring = req.search.split("?")[1];
+  }
+  const uri = url.split("?")[0];
+  console.log(`Simulating a request for: ${url}`);
+
+  const event = {};
+  const origin = {};
+  origin.custom = {};
+  // This always pretends to proceed and do a S3 lookup
+  origin.custom.domainName =
+    req.headers["ORIGIN_DOMAIN_NAME"] || "s3.fakey.fake";
+  const cf = {};
+  const headers = {};
+  headers.host = [{ value: req.hostname }];
+  for (const [key, value] of Object.entries(req.headers)) {
+    headers[key] = [{ value }];
+  }
+  cf.request = {
+    uri,
+    querystring,
+    origin,
+    headers,
+  };
+
+  event.Records = [{ cf }];
+  // console.log("EVENT...");
+  // console.log(event);
+  // console.log(JSON.stringify(event, null, 3));
+  const handle = await handler(event);
+  if (handle === cf.request) {
+    // The request is allowed to pass through.
+    // The URL might have been mutated.
+    const msg = `Looking up ${kleur.white().bold(handle.uri)} in S3!`;
+    console.log(msg);
+    // This is unrealistic but helpful if you want to write some integration
+    // tests against this test server if you're interesting in seeing that
+    // would happen next.
+    res.setHeader("X-S3-Lookup", handle.uri);
+    res.end(`${msg}\n`);
+  } else if (handle.status) {
+    // It's a redirect.
+    // console.log(JSON.stringify(handle, null, 3));
+    let path = null;
+    for (const headers of Object.values(handle.headers)) {
+      for (const header of headers) {
+        res.setHeader(header.key, header.value);
+        if (header.key.toLowerCase() === "location") {
+          path = header.value;
+        }
+      }
+    }
+    console.log(
+      `${
+        handle.status === 302
+          ? kleur.green(handle.status)
+          : kleur.yellow(handle.status)
+      } redirect to ${kleur.bold(path)}`
+    );
+    res.statusCode = handle.status;
+    res.setHeader("Location", path);
+    res.end("");
+  } else {
+    console.warn(JSON.stringify(handle, null, 3));
+    res.statusCode = 500;
+    res.end("Unrecognized handler response");
+  }
+}
+
+polka()
+  .get("/ping", ping)
+  .head("/ping", ping)
+  .get("/*", catchall)
+  .head("/*", catchall)
+  .listen(PORT, (err) => {
+    if (err) {
+      throw err;
+    }
+    console.log(
+      `content-origin-request simulator started on http://localhost:${PORT}`
+    );
+  });