add some docs

Makefile

@@ -1,6 +1,9 @@
 test: lint
 	NODE_ENV=testing ./node_modules/.bin/_mocha ./test/**/*.js
 
+test-bail: lint
+	NODE_ENV=testing ./node_modules/.bin/_mocha ./test/**/*.js --bail
+
 lint:
 	./node_modules/.bin/eslint ./src/**/*.js
 

README.md

@@ -37,147 +37,4 @@ Mutation testing is different from and generally more comprehensive than code co
 ## Stability Disclaimer
 This project is in very early development, and should be considered _experimental_. It "works", but there are a ton of issues to address and core features to add. The configuration API (the only public interface to `perturb`) is subject to breaking changes. Any and all ideas, suggestions, contributions welcome. Just open an [issue](https://github.com/nickb1080/perturb/issues) or a [pull request](https://github.com/nickb1080/perturb/pulls).
 
-## Compile-to-JS Languages
-I interested in eventually providing an entry point for extensions which mutate compile-to-JS languages like CoffeeScript. You can of course already compile your code to JavaScript and then mutate it, but it will be difficult to track down exactly what mutations aren't satisfied. Any contributions in this area are very welcome.
-
-## Known Issues
-1.) **Mocha only** -- Perturb only supports Mocha as a test runner at this point. PRs welcome for adding other test runners. I plan to add support for [tap](http://testanything.org/)-[compliant](https://github.com/isaacs/node-tap)  [harnesses](https://github.com/substack/tape) eventually. If you want another test runner, open an issue, or better, yet a PR.
-
-2.) **Infinite loops** -- mutating loop constructs like `while` and `for` is tricky, given the many ways in which these loops might terminate. First and foremost, the mutation to swap `++` and `--` is currently disabled, because it will break most `for`-loops. However, infinite loops may still occur in some cases. If running `perturb` seems to hang, this is the likely culprit. Luckily, hard synchronous loops are somewhat rare; blowing the call stack with bad recursion is probably a more likely mutation result, and that'll should just throw. If you hit a loop and are able to figure it out, please [open an issue](https://github.com/nickb1080/perturb/issues). One potential fix for this is to run each mutant in a child process, killing the process if it hangs for a certain duration. It's not clear that the complexity and (probable) performance hit would be worth it for a case which, so far, seems fairly rare.
-
-3.) **Equivalent mutations** -- there may be mutations that are functionally identical. For instance, given this source code:
-
-```js
-if (nextValue === value) {
-  // ...
-}
-```
-
-We will generate two functionally identical mutants:
-
-```js
-// invert conditional test
-if (!(nextValue === value)) {
-  // ...
-}
-
-// swap binary operators
-if (nextValue !== value) {
-  // ...
-}
-```
-
-This could skew metrics on kill rate since a single fix will kill both mutants. That said, it doesn't seem like a real problem so far.
-
-## API
-```js
-// default config
-{
-  sharedParent: process.cwd(),
-  sourceDir: "lib",
-  testDir: "test",
-  sourceGlob: "/**/*.js",
-  testGlob: "/**/*.js",
-  tempTest: ".perturb-test",
-  tempSource: ".perturb-source",
-  reporter: defaultReporter,
-  matcher: function (sourceFile) 
-    return sourceFile.replace(".js", "-test.js");
-  }
-};
-```
-
-## CLI
-You can pass in any of the configuration parameters that are strings through the command line interface.
-
-`perturb --testGlob '/**/*-test.js' --testDir 'test-unit'"`
-
-## Interfaces
-Various configuration parameters allow you to pass in functions which will interact with internal data representations. They are documented here in [IDL](https://heycam.github.io/webidl/)
-
-### `PerturbReport`
-```idl
-interface PerturbReport {
-  attribute Metadata metadata;
-  attribute Config config;
-  attribute matches []Match;
-}
-```
-
-### `Meta`
-```idl
-interface Metadata {
-  attribute boolean errored;
-  attribute number duration;
-  attribute number matchCount;
-  attribute number mutantCount;
-  attribute number mutantKillCount;
-  attribute number killRate;
-}
-```
-
-### `Config`
-
-
-```idl
-interface Config {
-  attribute string sharedParent;
-  attribute string sourceDir;
-  attribute string sourceGlob;
-  attribute string sourceTemp;
-  attribute string testDir;
-  attribute string testGlob;
-  attribute string testTemp;
-  void matchReporter(Match match);
-  void mutantReporter(Mutant mutant);
-  void summaryReporter(Meta meta);
-}
-```
-
-### `Match`
-```idl
-interface Match {
-  attribute string sourceFile
-  attribute string testFile
-  attribute []MutantReport mutants
-}
-```
-
-### `MutantReport`
-```idl
-interface MutantReport {
-  attribute string loc
-  attribute MutantName name
-  attribute Diff diff
-  attribute String source
-  attribute []String passed
-  attribute []String failed
-}
-```
-
-### `MutantName`
-```idl
-enum MutantName {
-  "invertConditionalTest",
-  "reverseFunctionParameters",
-  "dropReturn",
-  "dropThrow",
-  "dropArrayElement",
-  "dropObjectProperty",
-  "tweakLiteralValue",
-  "swapLogicalOperators",
-  "swapBinaryOperators",
-  "dropUnaryOperator",
-  "dropMemberAssignment"
-}
-```
-
-### FAQ
-
-## Todo
-- [ ] Skip classes of mutations via settings / cmd line args
-- [ ] Skip mutation API via comments (or something else?) a la `jshint`
-- [ ] Targeted mutations of code fitting common patterns (particularly "classes")
-- [ ] .perturbrc
-- [ ] TAP runner
-- [ ] Catch infinite loops
+

doc/code-structure.md

@@ -0,0 +1,15 @@
+# Code Structure
+
+This project is structured as a series of independent steps which are orchestrated by the main function `perturb()` in `src/index.ts`. Each step's behavior can be customized through the use of plugins.
+
+1. Run the project's test suite to ensure it passes -- if it doesn't, mutation testing is pointless.
+
+2. Copy the source and test files into a new directory so we can safely work on those files without damaging the originals.
+
+3. Run the `makeMatches` function, using the active [matcher plugin]() on the source and test lists to get a list of of Match objects.
+
+4. [`chain`](http://ramdajs.com/0.21.0/docs/#chain) the matches through the `makeMutants` function, using the active [mutator plugins](), to generate a list of Mutant objects. (`chain` is also known as [`flatMap`](http://martinfowler.com/articles/collection-pipeline/flat-map.html) or [`mapcat`](https://clojuredocs.org/clojure.core/mapcat).)
+
+5. Map the mutant objects through a composition of the `runMutant` function (which delegates to the active [runner plugin]()) and the active [reporter plugin]()'s `onResult` method.
+
+6. Pass the list of run results to the active [reporter plugin]()'s `onFinish()` method.

doc/plugins.md

@@ -0,0 +1,73 @@
+# Plugins
+
+## Mutators
+**Multiple active**
+```typescript
+interface MutatorPlugin {
+  name: string;
+  nodeTypes: Array<string>;
+  filter?: (n: ESTree.Node): boolean;
+  mutator: (n: ESTree.Node): ESTree.Node;
+}
+```
+
+A mutator plugin describes a mutation to be applied to an AST node. Multiple mutator plugins can (and should) be active simultaneously. A mutator plugin has the following properties:
+
+- `name`: a unique string name
+- `nodeTypes`: an array of [node types]() the mutator may be run on
+- `filter`: an optional predicate function that filters out nodes that matched one of the provided node types.
+- `mutator`: a function that returns a new AST node to replace the old one. Despite the name, it **must not** actually mutate the old node by changing, adding, or removing it's properties. 
+
+## Matchers
+**One active**
+
+Matcher plugins come in two flavors, "generative" and "comparative". Comparative matchers are predicate functions which take a test file and a source file and return `true` when they match. 
+
+An example of comparative matching case might be when test files have names resembling source files, but with some kind of prefix or suffix indicating what they test. The tests on the Node.js [core libraries](https://github.com/nodejs/node/tree/master/test/parallel) often fit this pattern.
+
+By contrast, a generative matcher looks at the string path of a source file and returns the string path of a test file. This would be when, for instance, `/test/thing.js` is the test file for `/lib/thing.js`. Generative matchers imply a 1-1 mapping between test and source files.
+
+A matcher function, of either type, is initialized with a full configuration object, because it almost always needs to know the directory configuration values used.
+
+```
+interface MatcherPlugin {
+  name: string;
+  type: "comparative" | "generative",
+  makeMatcher: (cfg: PerturbConfig): GenerativeMatcher | ComparativeMatcher;
+}
+
+interface GenerativeMatcher {
+  (sourceFile: string): string;
+}
+
+interface ComparativeMatcher {
+  (sourceFile: string, testFile: string): boolean;
+}
+
+```
+
+## Runners
+**One active**
+```typescript
+interface RunnerPlugin {
+  name: string;
+  prepare: (m: Mutant): Promise<any>;
+  run: (m: Mutant): Promise<RunnerResult>;
+  cleanup: (m: Mutant, before?: any): Promise<void>;
+}
+```
+
+A runner plugin describes how to run a single mutation. As such, it needs to understand how to work with the test harness used by the project. A runner plugin has the following properties:
+
+- `name`: a unique string name
+- `prepare`: a function which sets up the run. In nearly every case, this function should write out the mutated source code to a file (unless you're doing something [exotic]()), but it can also do all kinds of other stuff, such as working with the `require` cache if the run is done in-process. It returns a promise, the result of which will be threaded back into the `cleanup` method.
+- `run`: a function which actually executes the tests over the given mutated source file. It returns a "RunnerResult", which is essentially a Mutant with an optional `error` field.
+- `cleanup`: a function which cleans up whatever side effects the `prepare` and `run` functions had. Often this involves rewriting the source file to its original value. It might also operate on the `require` cache or do other sorts of housekeeping.
+
+
+
+## Reporters
+**One active**
+
+## Skippers
+**Multiple active**

run.js

@@ -9,6 +9,7 @@ function run (perturb, which, cb) {
         rootDir: path.join(__dirname, ".."),
         sourceDir: "built",
         runner: "mocha",
+        testCmd: "make test-bail",
       };
       break;
 
@@ -47,8 +48,6 @@ function run (perturb, which, cb) {
       return results;
     }).catch(function (err) {
       console.log("fatal error in perturb");
-      console.log(err);
-      console.log(err.stack);
       process.exit(1);
     });
 }

src/index.ts

@@ -4,6 +4,8 @@
 
 const R = require("ramda");
 const Bluebird = require("bluebird");
+const { spawn } = require("child_process");
+const assert = require("assert");
 
 const getRunner = require("./runners");
 const getReporter = require("./reporters");
@@ -36,12 +38,11 @@ module.exports = function perturb (_cfg: PerturbConfig) {
   const matcher = getMatcher(cfg);
   const runner: RunnerPlugin = getRunner(cfg.runner);
   const reporter: ReporterPlugin = getReporter(cfg.reporter);
-
-  // this handler does all the interesting work on a single Mutant
-  const handler = makeMutantHandler(runner, reporter);
 
-  // first, set up the "shadow" file system that we'll work against
-  return Promise.resolve(setup())
+  // first run the tests, otherwise why bother at all?
+  return runTest(cfg)
+    // then, set up the "shadow" file system that we'll work against
+    .then(() => setup())
     // read those "shadow" directories and find the source and test files
     .then(() => paths())
     // use the matcher function to group {sourceFile, testFiles}
@@ -57,22 +58,12 @@ module.exports = function perturb (_cfg: PerturbConfig) {
       return tested;
     })
     // 
-    .then(matches => R.chain(makeMutants, matches))
-    .then(function (ms: Mutant[]) {
-      const noSource = ms.filter(m => m.mutatedSourceCode === "");
-      console.log("NO SOURCE:", noSource.length, "of total", ms.length);
-      return ms;
-    })
-    .then(function (ms: Mutant[]) {
-      // TODO -- right here we can serialize all the mutants before running them
-      // any reason we might want to do this?
-      // 
-      // this is the separation point between pure data and actually executing
-      // the tests against mutated source code
-      return ms;
-    })
-    // crank the mutants through the handler and gather the results
+    .then(R.chain(makeMutants))
+    .then(sanityCheckAndSideEffects)
+    // run the mutatnts and gather the results
     .then(function (ms: Mutant[]) {
+      // this handler does all the interesting work on a single Mutant
+      const handler = makeMutantHandler(runner, reporter);
       return Bluebird.mapSeries(ms, handler);
     })
     // run the final results handler, if supplied, then pass the results back
@@ -82,6 +73,10 @@ module.exports = function perturb (_cfg: PerturbConfig) {
         reporter.onFinish(rs);
       }
       return rs;
+    })
+    .catch(err => {
+      console.log("ERROR IN PERTURB MAIN CHAIN", err);
+      throw err;
     });
 }
 
@@ -107,3 +102,23 @@ function makeMutantHandler (runner: RunnerPlugin, reporter: ReporterPlugin) {
       })
   }
 }
+
+function runTest (cfg: PerturbConfig) {
+  return new Promise(function (resolve, reject) {
+    const [cmd, ...rest] = cfg.testCmd.split(/\s+/);
+    const child = spawn(cmd, rest);
+    child.stdout.pipe(process.stdout);
+    child.stderr.pipe(process.stderr);
+    child.on("close", function (code) {
+      code === 0 ? resolve() : reject(new Error(`Test command exited with non-zero code: ${code}`));
+    });
+  });
+}
+
+// TODO -- what else? Any reason might want to serialize mutants here?
+function sanityCheckAndSideEffects (ms: Mutant[]) {
+  ms.forEach(function (m: Mutant) {
+    assert.notEqual(m.mutatedSourceCode, "", "Mutated source code should not be empty.");
+  });
+  return ms;
+}

src/make-config.ts

@@ -6,6 +6,8 @@ import { PerturbConfig } from "./types";
 const CONFIG_FILE_NAME = ".perturbrc";
 
 const defaultConfig: PerturbConfig = {
+  testCmd: "npm test",
+
   projectRoot: process.cwd(),
   testDir: "test",
   sourceDir: "src",

src/matchers/base-comparative.ts

@@ -17,7 +17,6 @@ module.exports = <ComparativeMatcherPlugin>{
       const perturbRoot = path.join(c.projectRoot, c.perturbDir);
       const sourceName = sourceFile.split(path.join(perturbRoot, c.sourceDir)).pop();
       const testName = testFile.split(path.join(perturbRoot, c.testDir)).pop();
-      console.log("SOURCE NAME", sourceName, "TEST NAME", testName)
       return sourceName === testName;
     };
   }