From 614eb121ff6066e7fa72ec080671dedcbc625a3a Mon Sep 17 00:00:00 2001 From: James Talmage Date: Wed, 30 Dec 2015 02:19:15 -0500 Subject: [PATCH] Transpile test files in the main process: Squashed commits: transpile in the main thread drop unused dependencies incorporate PR feedback add tests add unit tests --- api.js | 15 ++++- cli.js | 4 +- lib/caching-precompiler.js | 94 +++++++++++++++++++++++++++++++ lib/test-worker.js | 92 +++++++----------------------- package.json | 12 ++-- test/api.js | 77 ++++++++++++++++++++++++- test/caching-precompiler.js | 50 ++++++++++++++++ test/fixture/caching/package.json | 4 ++ test/fixture/caching/test.js | 3 + test/fork.js | 24 ++++---- test/hooks.js | 12 +++- 11 files changed, 293 insertions(+), 94 deletions(-) create mode 100644 lib/caching-precompiler.js create mode 100644 test/caching-precompiler.js create mode 100644 test/fixture/caching/package.json create mode 100644 test/fixture/caching/test.js diff --git a/api.js b/api.js index 9e5d36124..50462a21c 100644 --- a/api.js +++ b/api.js @@ -8,11 +8,15 @@ var Promise = require('bluebird'); var figures = require('figures'); var globby = require('globby'); var chalk = require('chalk'); +var objectAssign = require('object-assign'); var commondir = require('commondir'); var resolveCwd = require('resolve-cwd'); var AvaError = require('./lib/ava-error'); var fork = require('./lib/fork'); var formatter = require('./lib/enhance-assert').formatter(); +var CachingPrecompiler = require('./lib/caching-precompiler'); +var uniqueTempDir = require('unique-temp-dir'); +var findCacheDir = require('find-cache-dir'); function Api(files, options) { if (!(this instanceof Api)) { @@ -44,7 +48,10 @@ util.inherits(Api, EventEmitter); module.exports = Api; Api.prototype._runFile = function (file) { - return fork(file, this.options) + var options = objectAssign({}, this.options, { + precompiled: this.precompiler.generateHashForFile(file) + }); + return fork(file, options) .on('stats', this._handleStats) .on('test', this._handleTest) .on('unhandledRejections', this._handleRejections) @@ -137,6 +144,12 @@ Api.prototype.run = function () { return Promise.reject(new AvaError('Couldn\'t find any files to test')); } + var cacheEnabled = self.options.cacheEnabled !== false; + var cacheDir = (cacheEnabled && findCacheDir({name: 'ava', files: files})) || + uniqueTempDir(); + self.options.cacheDir = cacheDir; + self.precompiler = new CachingPrecompiler(cacheDir); + self.fileCount = files.length; self.base = path.relative('.', commondir('.', files)) + path.sep; diff --git a/cli.js b/cli.js index 860358820..2a3215dbf 100755 --- a/cli.js +++ b/cli.js @@ -43,6 +43,7 @@ var cli = meow([ ' --require Module to preload (Can be repeated)', ' --tap Generate TAP output', ' --verbose Enable verbose output', + ' --no-cache Disable the transpiler cache', '', 'Examples', ' ava', @@ -77,7 +78,8 @@ if (cli.flags.init) { var api = new Api(cli.input, { failFast: cli.flags.failFast, serial: cli.flags.serial, - require: arrify(cli.flags.require) + require: arrify(cli.flags.require), + cacheEnabled: cli.flags.cache !== false }); var logger = new Logger(); diff --git a/lib/caching-precompiler.js b/lib/caching-precompiler.js new file mode 100644 index 000000000..8e7de1bd0 --- /dev/null +++ b/lib/caching-precompiler.js @@ -0,0 +1,94 @@ +var cachingTransform = require('caching-transform'); +var fs = require('fs'); +var path = require('path'); +var md5Hex = require('md5-hex'); +var stripBom = require('strip-bom'); + +module.exports = CachingPrecompiler; + +function CachingPrecompiler(cacheDir) { + if (!(this instanceof CachingPrecompiler)) { + throw new Error('CachingPrecompiler must be called with new'); + } + this.cacheDir = cacheDir; + this.filenameToHash = {}; + this.transform = this._createTransform(); +} + +CachingPrecompiler.prototype._factory = function (cacheDir) { + // This factory method is only called once per process, and only as needed, to defer loading expensive dependencies. + var babel = require('babel-core'); + var convertSourceMap = require('convert-source-map'); + var presetStage2 = require('babel-preset-stage-2'); + var presetES2015 = require('babel-preset-es2015'); + var transformRuntime = require('babel-plugin-transform-runtime'); + + var powerAssert = this._createEspowerPlugin(babel); + + function buildOptions(filename, code) { + // Extract existing source maps from the code. + var sourceMap = convertSourceMap.fromSource(code) || convertSourceMap.fromMapFileSource(code, path.dirname(filename)); + + return { + presets: [presetStage2, presetES2015], + plugins: [powerAssert, transformRuntime], + filename: filename, + sourceMaps: true, + ast: false, + babelrc: false, + inputSourceMap: sourceMap && sourceMap.toObject() + }; + } + + return function (code, filename, hash) { + code = code.toString(); + var options = buildOptions(filename, code); + var result = babel.transform(code, options); + var mapFile = path.join(cacheDir, hash + '.map'); + fs.writeFileSync(mapFile, JSON.stringify(result.map)); + return result.code; + }; +}; + +CachingPrecompiler.prototype._createEspowerPlugin = function (babel) { + var createEspowerPlugin = require('babel-plugin-espower/create'); + var enhanceAssert = require('./enhance-assert'); + + // initialize power-assert + return createEspowerPlugin(babel, { + patterns: enhanceAssert.PATTERNS + }); +}; + +CachingPrecompiler.prototype._createTransform = function () { + return cachingTransform({ + factory: this._factory.bind(this), + cacheDir: this.cacheDir, + salt: new Buffer(JSON.stringify({ + 'babel-plugin-espower': require('babel-plugin-espower/package.json').version, + 'ava': require('../package.json').version, + 'babel-core': require('babel-core/package.json').version + })), + ext: '.js', + hash: this._hash.bind(this) + }); +}; + +CachingPrecompiler.prototype._hash = function (code, filename, salt) { + var hash = md5Hex([code, filename, salt]); + this.filenameToHash[filename] = hash; + return hash; +}; + +CachingPrecompiler.prototype.precompileFile = function (filename) { + if (!this.filenameToHash[filename]) { + this.transform(stripBom(fs.readFileSync(filename)), filename); + } + return this.filenameToHash[filename]; +}; + +CachingPrecompiler.prototype.generateHashForFile = function (filename) { + var hash = {}; + hash[filename] = this.precompileFile(filename); + return hash; +}; diff --git a/lib/test-worker.js b/lib/test-worker.js index 5409a4f8d..4fed7c607 100644 --- a/lib/test-worker.js +++ b/lib/test-worker.js @@ -1,16 +1,12 @@ 'use strict'; var path = require('path'); +var fs = require('fs'); var debug = require('debug')('ava'); -var pkgDir = require('pkg-dir').sync; -var hasha = require('hasha'); -var cacha = require('cacha'); var sourceMapSupport = require('source-map-support'); var opts = JSON.parse(process.argv[2]); var testPath = opts.file; -var cache = cacha(path.join(pkgDir(path.dirname(testPath)), 'node_modules', '.cache', 'ava')); - if (debug.enabled) { // Forward the `time-require` `--sorted` flag. // Intended for internal optimization tests only. @@ -38,73 +34,40 @@ sourceMapSupport.install({ if (sourceMapCache[source]) { return { url: source, - map: sourceMapCache[source] + map: fs.readFileSync(sourceMapCache[source], 'utf8') }; } } }); -var requireFromString = require('require-from-string'); var loudRejection = require('loud-rejection/api')(process); var serializeError = require('serialize-error'); var send = require('./send'); - -// if generators are not supported, use regenerator -var options = { - sourceMaps: true -}; +var installPrecompiler = require('require-precompiled'); +var cacheDir = opts.cacheDir; // check if test files required ava and show error, when they didn't exports.avaRequired = false; -// try to load an input source map for the test file, in case the file was -// already compiled once by the user -var inputSourceMap = sourceMapSupport.retrieveSourceMap(testPath); -if (inputSourceMap) { - // source-map-support returns the source map as a json-encoded string, but - // babel requires an actual object - options.inputSourceMap = JSON.parse(inputSourceMap.map); -} - -// include test file -var cachePath = hasha(cacheKey(testPath)); -var hashPath = cachePath + '_hash'; - -var prevHash = cache.getSync(hashPath, {encoding: 'utf8'}); -var currHash = hasha.fromFileSync(testPath); - -if (prevHash === currHash) { - var cached = JSON.parse(cache.getSync(cachePath)); - - sourceMapCache[testPath] = cached.map; - requireFromString(cached.code, testPath, { - appendPaths: module.paths - }); -} else { - var createEspowerPlugin = require('babel-plugin-espower/create'); - var babel = require('babel-core'); - - // initialize power-assert - var powerAssert = createEspowerPlugin(babel, { - patterns: require('./enhance-assert').PATTERNS - }); - - options.presets = [require('babel-preset-stage-2'), require('babel-preset-es2015')]; - options.plugins = [powerAssert, require('babel-plugin-transform-runtime')]; +installPrecompiler(function (filename) { + var precompiled = opts.precompiled[filename]; + if (precompiled) { + sourceMapCache[filename] = path.join(cacheDir, precompiled + '.map'); + return fs.readFileSync(path.join(cacheDir, precompiled + '.js'), 'utf8'); + } + return null; +}); - var transpiled = babel.transformFileSync(testPath, options); +// Modules need to be able to find `babel-runtime`, which is nested in our node_modules w/ npm@2 +var nodeModulesDir = path.join(__dirname, '../node_modules'); +var oldNodeModulesPaths = module.constructor._nodeModulePaths; +module.constructor._nodeModulePaths = function () { + var ret = oldNodeModulesPaths.apply(this, arguments); + ret.push(nodeModulesDir); + return ret; +}; - cache.setSync(hashPath, currHash); - cache.setSync(cachePath, JSON.stringify({ - code: transpiled.code, - map: transpiled.map - })); - - sourceMapCache[testPath] = transpiled.map; - requireFromString(transpiled.code, testPath, { - appendPaths: module.paths - }); -} +require(testPath); process.on('uncaughtException', function (exception) { send('uncaughtException', {exception: serializeError(exception)}); @@ -151,16 +114,3 @@ process.on('ava-teardown', function () { function exit() { send('teardown'); } - -function cacheKey(path) { - var key = path; - - key += require('../package.json').version; - key += require('babel-core/package.json').version; - key += require('babel-plugin-espower/package.json').version; - key += require('babel-plugin-transform-runtime/package.json').version; - key += require('babel-preset-stage-2/package.json').version; - key += require('babel-preset-es2015/package.json').version; - - return hasha(key); -} diff --git a/package.json b/package.json index e5e3a7a03..ac378e81a 100644 --- a/package.json +++ b/package.json @@ -85,44 +85,48 @@ "babel-preset-stage-2": "^6.3.13", "babel-runtime": "^6.3.19", "bluebird": "^3.0.0", - "cacha": "^1.0.3", + "caching-transform": "^1.0.0", "chalk": "^1.0.0", "co-with-promise": "^4.6.0", "commondir": "^1.0.1", + "convert-source-map": "^1.1.2", "core-assert": "^0.1.0", "debug": "^2.2.0", "deeper": "^2.1.0", "empower-core": "^0.2.0", "figures": "^1.4.0", + "find-cache-dir": "^0.1.1", "fn-name": "^2.0.0", "globby": "^4.0.0", - "hasha": "^2.0.2", "is-generator-fn": "^1.0.0", "is-observable": "^0.1.0", "is-promise": "^2.1.0", "log-update": "^1.0.2", "loud-rejection": "^1.2.0", "max-timeout": "^1.0.0", + "md5-hex": "^1.2.0", "meow": "^3.6.0", "object-assign": "^4.0.1", "observable-to-promise": "^0.1.0", - "pkg-dir": "^1.0.0", "plur": "^2.0.0", "power-assert-formatter": "^1.3.0", "power-assert-renderers": "^0.1.0", "pretty-ms": "^2.0.0", - "require-from-string": "^1.1.0", + "require-precompiled": "^0.1.0", "resolve-cwd": "^1.0.0", "serialize-error": "^1.1.0", "set-immediate-shim": "^1.0.1", "source-map-support": "^0.4.0", + "strip-bom": "^2.0.0", "time-require": "^0.1.2", + "unique-temp-dir": "^1.0.0", "update-notifier": "^0.6.0" }, "devDependencies": { "coveralls": "^2.11.4", "delay": "^1.3.0", "get-stream": "^1.1.0", + "rimraf": "^2.5.0", "nyc": "^5.1.0", "signal-exit": "^2.1.2", "sinon": "^1.17.2", diff --git a/test/api.js b/test/api.js index 8b4e22b8a..052e10e80 100644 --- a/test/api.js +++ b/test/api.js @@ -1,6 +1,8 @@ 'use strict'; var path = require('path'); var figures = require('figures'); +var rimraf = require('rimraf'); +var fs = require('fs'); var test = require('tap').test; var Api = require('../api'); @@ -217,7 +219,7 @@ test('uncaught exception will throw an error', function (t) { test('stack traces for exceptions are corrected using a source map file', function (t) { t.plan(4); - var api = new Api([path.join(__dirname, 'fixture/source-map-file.js')]); + var api = new Api([path.join(__dirname, 'fixture/source-map-file.js')], {cacheEnabled: true}); api.on('error', function (data) { t.match(data.message, /Thrown by source-map-fixtures/); @@ -231,10 +233,44 @@ test('stack traces for exceptions are corrected using a source map file', functi }); }); -test('stack traces for exceptions are corrected using a source map, taking an initial source map for the test file into account', function (t) { +test('stack traces for exceptions are corrected using a source map file (cache off)', function (t) { t.plan(4); - var api = new Api([path.join(__dirname, 'fixture/source-map-initial.js')]); + var api = new Api([path.join(__dirname, 'fixture/source-map-file.js')], {cacheEnabled: false}); + + api.on('error', function (data) { + t.match(data.message, /Thrown by source-map-fixtures/); + t.match(data.stack, /^.*?at.*?run\b.*source-map-fixtures.src.throws.js:1.*$/m); + t.match(data.stack, /^.*?at\b.*source-map-file.js:11.*$/m); + }); + + api.run() + .then(function () { + t.is(api.passCount, 1); + }); +}); + +test('stack traces for exceptions are corrected using a source map, taking an initial source map for the test file into account (cache on)', function (t) { + t.plan(4); + + var api = new Api([path.join(__dirname, 'fixture/source-map-initial.js')], {cacheEnabled: true}); + + api.on('error', function (data) { + t.match(data.message, /Thrown by source-map-fixtures/); + t.match(data.stack, /^.*?at.*?run\b.*source-map-fixtures.src.throws.js:1.*$/m); + t.match(data.stack, /^.*?at\b.*source-map-initial-input.js:7.*$/m); + }); + + api.run() + .then(function () { + t.is(api.passCount, 1); + }); +}); + +test('stack traces for exceptions are corrected using a source map, taking an initial source map for the test file into account (cache off)', function (t) { + t.plan(4); + + var api = new Api([path.join(__dirname, 'fixture/source-map-initial.js')], {cacheEnabled: false}); api.on('error', function (data) { t.match(data.message, /Thrown by source-map-fixtures/); @@ -404,3 +440,38 @@ test('power-assert support', function (t) { ); }); }); + +test('caching is enabled by default', function (t) { + t.plan(3); + rimraf.sync(path.join(__dirname, 'fixture/caching/node_modules')); + var api = new Api([path.join(__dirname, 'fixture/caching/test.js')]); + + api.run() + .then(function () { + var files = fs.readdirSync(path.join(__dirname, 'fixture/caching/node_modules/.cache/ava')); + t.is(files.length, 2); + t.is(files.filter(endsWithJs).length, 1); + t.is(files.filter(endsWithMap).length, 1); + t.end(); + }); + + function endsWithJs(filename) { + return /\.js$/.test(filename); + } + + function endsWithMap(filename) { + return /\.js$/.test(filename); + } +}); + +test('caching can be disabled', function (t) { + t.plan(1); + rimraf.sync(path.join(__dirname, 'fixture/caching/node_modules')); + var api = new Api([path.join(__dirname, 'fixture/caching/test.js')], {cacheEnabled: false}); + + api.run() + .then(function () { + t.false(fs.existsSync(path.join(__dirname, 'fixture/caching/node_modules/.cache/ava'))); + t.end(); + }); +}); diff --git a/test/caching-precompiler.js b/test/caching-precompiler.js new file mode 100644 index 000000000..2e1a0edcb --- /dev/null +++ b/test/caching-precompiler.js @@ -0,0 +1,50 @@ +'use strict'; +var fs = require('fs'); +var path = require('path'); +var test = require('tap').test; +var uniqueTempDir = require('unique-temp-dir'); + +var CachingPrecompiler = require('../lib/caching-precompiler'); +var createCachingPrecompiler = CachingPrecompiler; + +function fixture(name) { + return path.join(__dirname, 'fixture', name); +} + +function endsWithJs(filename) { + return /\.js$/.test(filename); +} + +function endsWithMap(filename) { + return /\.js$/.test(filename); +} + +test('creation with new', function (t) { + var tempDir = uniqueTempDir(); + var precompiler = new CachingPrecompiler(tempDir); + t.is(precompiler.cacheDir, tempDir); + t.end(); +}); + +test('creation without new throws', function (t) { + t.throws(function () { + createCachingPrecompiler(uniqueTempDir()); + }); + t.end(); +}); + +test('adds files and source maps to the cache directory as needed', function (t) { + var tempDir = uniqueTempDir(); + var precompiler = new CachingPrecompiler(tempDir); + + t.false(fs.existsSync(tempDir), 'cache directory is not created before it is needed'); + + precompiler.precompileFile(fixture('es2015.js')); + t.true(fs.existsSync(tempDir), 'cache directory is lazily created'); + + var files = fs.readdirSync(tempDir); + t.is(files.length, 2); + t.is(files.filter(endsWithJs).length, 1, 'one .js file is saved to the cache'); + t.is(files.filter(endsWithMap).length, 1, 'one .map file is saved to the cache'); + t.end(); +}); diff --git a/test/fixture/caching/package.json b/test/fixture/caching/package.json new file mode 100644 index 000000000..daa8ae391 --- /dev/null +++ b/test/fixture/caching/package.json @@ -0,0 +1,4 @@ +{ + "name": "application-name", + "version": "0.0.1" +} diff --git a/test/fixture/caching/test.js b/test/fixture/caching/test.js new file mode 100644 index 000000000..e50e27479 --- /dev/null +++ b/test/fixture/caching/test.js @@ -0,0 +1,3 @@ +import test from '../../../' + +test(t => t.true(2 + 2 === 4)); diff --git a/test/fork.js b/test/fork.js index 7fb142852..e67def95d 100644 --- a/test/fork.js +++ b/test/fork.js @@ -1,7 +1,17 @@ 'use strict'; var path = require('path'); var test = require('tap').test; -var fork = require('../lib/fork.js'); +var _fork = require('../lib/fork.js'); +var CachingPrecompiler = require('../lib/caching-precompiler'); +var cacheDir = path.join(__dirname, '../node_modules/.cache/ava'); +var precompiler = new CachingPrecompiler(cacheDir); + +function fork(testPath) { + return _fork(testPath, { + cacheDir: cacheDir, + precompiled: precompiler.generateHashForFile(testPath) + }); +} function fixture(name) { return path.join(__dirname, 'fixture', name); @@ -33,18 +43,6 @@ test('resolves promise with tests info', function (t) { }); }); -test('rejects on error and streams output', function (t) { - t.plan(2); - - fork(fixture('broken.js'), {silent: true}) - .run() - .catch(function (err) { - t.ok(err); - t.match(err.message, /exited with a non-zero exit code: \d/); - t.end(); - }); -}); - test('exit after tests are finished', function (t) { t.plan(2); diff --git a/test/hooks.js b/test/hooks.js index fc8dec7c9..cc6703b13 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -2,7 +2,17 @@ var path = require('path'); var test = require('tap').test; var Runner = require('../lib/runner'); -var fork = require('../lib/fork'); +var _fork = require('../lib/fork.js'); +var CachingPrecompiler = require('../lib/caching-precompiler'); +var cacheDir = path.join(__dirname, '../node_modules/.cache/ava'); +var precompiler = new CachingPrecompiler(cacheDir); + +function fork(testPath) { + return _fork(testPath, { + cacheDir: cacheDir, + precompiled: precompiler.generateHashForFile(testPath) + }); +} test('before', function (t) { t.plan(1);