Adds static asset support (#522)

* adds static asset support #518

* fixes assets manifest merging; fixes server public assets in dev; fixes static starter script order

alpha.2 published

CHANGELOG.md

@@ -5,6 +5,16 @@
 
 ## Master
 
+## 0.9.0-alpha.2 08/08/17
+
+- Fixes assets manifest merging by only merging server public assets
+- Fixes 404'ing server public assets in `dev` mode
+- Fixes static starter script order
+
+## 0.9.0-alpha.1 08/03/17
+
+- Adds static asset support [518](https://github.com/NYTimes/kyt/issues/518)
+
 ## 0.8.0 08/03/17
 
 [0.7.x-0.8.0 Migration guide](/docs/migration-guides/0.7-0.8.md).

e2e_tests/fixtures/build-default/src/client/index.js

@@ -1,2 +1,6 @@
 
+import '../public/img.jpg';
+import '../public/script.js';
+import '../assets/file.ico';
+
 module.exports = { client: true };

e2e_tests/fixtures/build-default/src/public/script.js

@@ -0,0 +1 @@
+x = 1;

e2e_tests/fixtures/build-default/src/server/index.js

@@ -1,2 +1,4 @@
 
+import '../public/img-server.png';
+
 module.exports = { server: true };

e2e_tests/tests/kyt-build.test.js

@@ -1,5 +1,6 @@
 const shell = require('shelljs');
 const util = require('../fixtures/util');
+const fs = require('fs');
 
 shell.config.silent = true;
 
@@ -23,6 +24,21 @@ describe('kyt build', () => {
     expect(shell.ls('build/public/main-*.js').code).toBe(0);
     expect(shell.ls('build/public/vendor-*.js').code).toBe(0);
 
+    // Should fingerprint client and server assets
+    expect(shell.ls('build/public/img-server-*.png').code).toBe(0);
+    expect(shell.ls('build/public/img-*.jpg').code).toBe(0);
+    expect(shell.ls('build/public/script-*.js').code).toBe(0);
+    expect(shell.ls('build/public/file-*.ico').code).toBe(0);
+
+    // Should produce asset manifest mappings for client and server assets and bundles
+    const manifest = JSON.parse(fs.readFileSync('build/publicAssets.json', 'utf8'));
+    expect(manifest['img-server.png']).toMatch(/img-server-.*\.png/);
+    expect(manifest['img.jpg']).toMatch(/img-.*\.jpg/);
+    expect(manifest['script.js']).toMatch(/script-.*\.js/);
+    expect(manifest['file.ico']).toMatch(/file-.*\.ico/);
+    expect(manifest['main.js']).toMatch(/main-.*\.js/);
+    expect(manifest['vendor.js']).toMatch(/vendor-.*\.js/);
+
     expect(output.code).toBe(0);
   });
 

packages/kyt-core/cli/actions/__tests__/dev.test.js

@@ -102,7 +102,7 @@ describe('dev', () => {
     // start client
     assert.ok(devMiddleware.mock.calls.length > 0, 'should call devMiddleware');
     assert.ok(express.mock.calls.length > 0, 'should instantiate express');
-    assert.equal(express.use.mock.calls.length, 2, 'should set up two express middlewares');
+    assert.equal(express.use.mock.calls.length, 5, 'should set up two express middlewares');
     assert.deepEqual(
       express.listen.mock.calls,
       [[port, hostname]],

packages/kyt-core/cli/actions/dev.js

@@ -3,6 +3,7 @@
 const path = require('path');
 const chokidar = require('chokidar');
 const express = require('express');
+const cors = require('cors');
 const shell = require('shelljs');
 const devMiddleware = require('webpack-dev-middleware');
 const hotMiddleware = require('webpack-hot-middleware');
@@ -13,7 +14,7 @@ const logger = require('kyt-utils/logger');
 const ifPortIsFreeDo = require('../../utils/ifPortIsFreeDo');
 const buildConfigs = require('../../utils/buildConfigs');
 const webpackCompiler = require('../../utils/webpackCompiler');
-const { buildPath, serverSrcPath, publicSrcPath } = require('kyt-utils/paths')();
+const { buildPath, serverSrcPath, publicSrcPath, publicBuildPath } = require('kyt-utils/paths')();
 
 module.exports = (config, flags) => {
   logger.start('Starting development build...');
@@ -43,12 +44,21 @@ module.exports = (config, flags) => {
     const webpackDevMiddleware = devMiddleware(clientCompiler, devOptions);
 
     if (!hasServer) {
+      // For client only apps, use history api fallback
+      // so all routes will be served from index.html.
       app.use(history());
-      app.use(express.static(publicSrcPath));
+    } else {
+      // Setup a static proxy for static assets that
+      // are imported by file-loader by the server.
+      app.use(cors());
+      app.use(express.static(publicBuildPath));
     }
+
+    // Setup a static proxy for the public source directory.
+    app.use(express.static(publicSrcPath));
+
     app.use(webpackDevMiddleware);
     app.use(hotMiddleware(clientCompiler));
-
     app.listen(clientURL.port, clientURL.hostname);
   };
 

packages/kyt-core/config/webpack.base.js

@@ -3,13 +3,24 @@
 // https://github.com/survivejs/webpack-merge
 
 const path = require('path');
+const fs = require('fs');
 const webpack = require('webpack');
 const shell = require('shelljs');
-const { buildPath, userNodeModulesPath, userBabelrcPath } = require('kyt-utils/paths')();
+const merge = require('lodash.merge');
+const WebpackAssetsManifest = require('webpack-assets-manifest');
+const {
+  buildPath,
+  userNodeModulesPath,
+  userBabelrcPath,
+  publicSrcPath,
+} = require('kyt-utils/paths')();
 const fileExtensions = require('./fileExtensions');
 
+let clientAssets;
+
 module.exports = options => {
   const hasBabelrc = shell.test('-f', userBabelrcPath);
+  const assetsFilePath = path.join(buildPath, options.clientAssetsFile);
 
   return {
     node: {
@@ -43,25 +54,67 @@ module.exports = options => {
           ),
         },
       }),
+
+      new WebpackAssetsManifest({
+        output: assetsFilePath,
+        space: 2,
+        fileExtRegex: /\.\w{2,4}\.(?:map|gz)$|\.\w+$/i,
+        writeToDisk: options.type === 'client',
+        done: manifest => {
+          // This plugin's `merge` doesn't work as expected. The "done" callback
+          // gets called for the client and server asset builds, in that order.
+          if (options.type === 'client') {
+            // Save the client assets for merging later.
+            clientAssets = manifest.toJSON();
+          } else {
+            // Merge the server assets into the client assets and write the result to disk.
+            const assets = merge({}, clientAssets, manifest.toJSON());
+            fs.writeFile(assetsFilePath, JSON.stringify(assets, null, '  '), 'utf8');
+          }
+        },
+        customize: (key, value) => {
+          const prependPublicPath = asset => `${options.publicPath || ''}${asset}`;
+          const removePublicDir = asset => asset.replace(/(.*)?public\//, '');
+
+          // Server asset files have "../public" prepended to them
+          // (see file-loader `outputPath`). We need to remove that.
+          if (options.type === 'server') {
+            if (value.startsWith('../public')) {
+              key = removePublicDir(key);
+              value = prependPublicPath(removePublicDir(value));
+            } else {
+              return false;
+            }
+          } else {
+            value = prependPublicPath(value);
+          }
+
+          return { key, value };
+        },
+      }),
     ],
 
     module: {
       rules: [
         {
-          test: /\.html$/,
-          loader: 'file-loader?name=[name].[ext]',
-        },
-        {
-          test: new RegExp(fileExtensions),
-          loader: 'url-loader',
+          test: asset => {
+            const extensions = new RegExp(fileExtensions);
+            const jsCSSExtensions = new RegExp('\\.(js|css)$');
+            const isFile = extensions.test(asset);
+            const isJSOrCSSInPublic = asset.includes('/src/public/') && jsCSSExtensions.test(asset);
+            return isFile || isJSOrCSSInPublic;
+          },
+          loader: 'file-loader',
           options: {
-            limit: 20000,
+            name: '[name]-[hash].[ext]',
+            publicPath: options.publicPath,
+            outputPath: asset => (options.type === 'server' ? `../public/${asset}` : asset),
           },
         },
         {
           test: /\.(js|jsx)$/,
           loader: 'babel-loader',
-          exclude: [/node_modules/, buildPath],
+          exclude: [/node_modules/, buildPath, publicSrcPath],
           // babel configuration should come from presets defined in the user's
           // .babelrc, unless there's a specific reason why it has to be put in
           // the webpack loader options

packages/kyt-core/config/webpack.dev.client.js

@@ -1,17 +1,19 @@
 // Development webpack config for client code
 
 const webpack = require('webpack');
-const WebpackAssetsManifest = require('webpack-assets-manifest');
 const clone = require('lodash.clonedeep');
-const path = require('path');
-const { clientSrcPath, buildPath, assetsBuildPath } = require('kyt-utils/paths')();
+const { clientSrcPath, assetsBuildPath, publicSrcPath } = require('kyt-utils/paths')();
 const postcssLoader = require('../utils/getPostcssLoader');
 
 const cssStyleLoaders = [
   'style-loader',
   {
     loader: 'css-loader',
-    options: { modules: true, sourceMap: true, localIdentName: '[name]-[local]--[hash:base64:5]' },
+    options: {
+      modules: true,
+      sourceMap: true,
+      localIdentName: '[name]-[local]--[hash:base64:5]',
+    },
   },
   postcssLoader,
 ];
@@ -57,10 +59,12 @@ module.exports = options => {
         {
           test: /\.css$/,
           use: cssStyleLoaders,
+          exclude: [publicSrcPath],
         },
         {
           test: /\.scss$/,
           use: clone(cssStyleLoaders).concat('sass-loader'),
+          exclude: [publicSrcPath],
         },
       ],
     },
@@ -70,15 +74,6 @@ module.exports = options => {
 
       new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1 }),
 
-      new WebpackAssetsManifest({
-        output: path.join(buildPath, options.clientAssetsFile),
-        space: 2,
-        writeToDisk: true,
-        fileExtRegex: /\.\w{2,4}\.(?:map|gz)$|\.\w+$/i,
-        merge: false,
-        publicPath: options.publicPath,
-      }),
-
       new webpack.HotModuleReplacementPlugin(),
     ],
   };

packages/kyt-core/config/webpack.dev.server.js

@@ -3,13 +3,16 @@
 const webpack = require('webpack');
 const nodeExternals = require('webpack-node-externals');
 const clone = require('lodash.clonedeep');
-const { serverSrcPath, serverBuildPath } = require('kyt-utils/paths')();
+const { serverSrcPath, serverBuildPath, publicSrcPath } = require('kyt-utils/paths')();
 const postcssLoader = require('../utils/getPostcssLoader');
 
 const cssStyleLoaders = [
   {
     loader: 'css-loader/locals',
-    options: { modules: true, localIdentName: '[name]-[local]--[hash:base64:5]' },
+    options: {
+      modules: true,
+      localIdentName: '[name]-[local]--[hash:base64:5]',
+    },
   },
   postcssLoader,
 ];
@@ -41,10 +44,12 @@ module.exports = options => ({
       {
         test: /\.css$/,
         use: cssStyleLoaders,
+        exclude: [publicSrcPath],
       },
       {
         test: /\.scss$/,
         use: clone(cssStyleLoaders).concat('sass-loader'),
+        exclude: [publicSrcPath],
       },
     ],
   },

packages/kyt-core/config/webpack.prod.client.js

@@ -2,11 +2,9 @@
 
 const webpack = require('webpack');
 const ExtractTextPlugin = require('extract-text-webpack-plugin');
-const WebpackAssetsManifest = require('webpack-assets-manifest');
 const clone = require('lodash.clonedeep');
 const postcssLoader = require('../utils/getPostcssLoader');
-const { clientSrcPath, assetsBuildPath, buildPath } = require('kyt-utils/paths')();
-const path = require('path');
+const { clientSrcPath, assetsBuildPath, publicSrcPath } = require('kyt-utils/paths')();
 
 const cssStyleLoaders = [
   {
@@ -44,13 +42,15 @@ module.exports = options => ({
           fallback: 'style-loader',
           use: cssStyleLoaders,
         }),
+        exclude: [publicSrcPath],
       },
       {
         test: /\.scss$/,
         loader: ExtractTextPlugin.extract({
           fallback: 'style-loader',
           use: clone(cssStyleLoaders).concat('sass-loader'),
         }),
+        exclude: [publicSrcPath],
       },
     ],
   },
@@ -77,15 +77,6 @@ module.exports = options => ({
       sourceMap: true,
     }),
 
-    new WebpackAssetsManifest({
-      output: path.join(buildPath, options.clientAssetsFile),
-      space: 2,
-      writeToDisk: true,
-      fileExtRegex: /\.\w{2,4}\.(?:map|gz)$|\.\w+$/i,
-      merge: false,
-      publicPath: options.publicPath,
-    }),
-
     // Modules should get deterministic ids so that they don't change between builds
     new webpack.HashedModuleIdsPlugin(),
 

packages/kyt-core/config/webpack.prod.server.js

@@ -3,7 +3,7 @@
 const webpack = require('webpack');
 const nodeExternals = require('webpack-node-externals');
 const clone = require('lodash.clonedeep');
-const { serverSrcPath, serverBuildPath } = require('kyt-utils/paths')();
+const { serverSrcPath, serverBuildPath, publicSrcPath } = require('kyt-utils/paths')();
 const postcssLoader = require('../utils/getPostcssLoader');
 
 const cssStyleLoaders = [
@@ -44,10 +44,12 @@ module.exports = options => ({
       {
         test: /\.css$/,
         use: cssStyleLoaders,
+        exclude: [publicSrcPath],
       },
       {
         test: /\.scss$/,
         use: clone(cssStyleLoaders).concat('sass-loader'),
+        exclude: [publicSrcPath],
       },
     ],
   },

packages/kyt-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kyt",
-  "version": "0.8.0",
+  "version": "0.9.0-alpha.2",
   "description": "kyt is a toolkit that encapsulates and manages the configuration for web apps.",
   "author": "NYTimes",
   "license": "Apache-2.0",
@@ -25,6 +25,7 @@
     "chokidar": "1.7.0",
     "commander": "2.9.0",
     "connect-history-api-fallback": "1.3.0",
+    "cors": "2.8.4",
     "css-loader": "0.28.2",
     "detect-port": "1.0.1",
     "eslint": "3.19.0",
@@ -69,7 +70,7 @@
     "temp": "0.8.3",
     "url-loader": "0.5.8",
     "webpack": "3.1.0",
-    "webpack-assets-manifest": "0.6.2",
+    "webpack-assets-manifest": "0.7.0",
     "webpack-dev-middleware": "1.11.0",
     "webpack-dev-server": "2.5.1",
     "webpack-hot-middleware": "2.18.2",