From e1a5d8ae9272d1d7f2c3156c11cb46f24370d947 Mon Sep 17 00:00:00 2001
From: "Daniel J. Hofmann" <daniel@trvx.org>
Date: Tue, 20 Sep 2016 15:00:51 +0200
Subject: [PATCH] Revert #2795: Rewrite cucumber test caching, Tests Failing At
 Random

This reverts commit 7d124ce54d55ddba12cc8af05b85377ac7da8000.
---
 cucumber.js                                   |  10 +-
 features/car/traffic_speeds.feature           |   9 +-
 features/car/traffic_turn_penalties.feature   |   4 +-
 features/guidance/anticipate-lanes.feature    |   4 +-
 features/guidance/turn-lanes.feature          |   2 +-
 features/lib/hash.js                          |  31 ---
 features/lib/osrm_loader.js                   | 169 ------------
 features/lib/table_diff.js                    |  54 ----
 features/lib/try_connect.js                   |  13 -
 features/lib/utils.js                         |  17 --
 features/options/contract/datasources.feature |  10 +-
 features/options/contract/files.feature       |  13 +-
 features/options/contract/help.feature        |   8 +-
 features/options/contract/invalid.feature     |   4 +-
 features/options/contract/version.feature     |   4 +-
 features/options/extract/files.feature        |  14 +-
 features/options/extract/help.feature         |   6 +-
 features/options/extract/invalid.feature      |   4 +-
 features/options/extract/version.feature      |   4 +-
 features/options/routed/files.feature         |   2 +-
 features/options/routed/help.feature          |   6 +-
 features/options/routed/invalid.feature       |   8 +-
 features/options/routed/version.feature       |   4 +-
 features/raster/extract.feature               |  19 +-
 features/raster/weights.feature               |  12 +-
 features/step_definitions/data.js             |  60 ++--
 features/step_definitions/distance_matrix.js  |   8 +
 features/step_definitions/hooks.js            |  18 ++
 features/step_definitions/matching.js         |   1 +
 features/step_definitions/nearest.js          |   8 +
 features/step_definitions/options.js          |  56 ++--
 features/step_definitions/requests.js         |   2 +-
 features/step_definitions/routability.js      |   8 +-
 features/step_definitions/trip.js             |   9 +
 features/{lib/osm.js => support/build_osm.js} |   8 +-
 features/support/cache.js                     | 184 -------------
 features/support/config.js                    | 127 +++++++++
 features/support/data.js                      | 256 +++++++++++++-----
 features/support/data_classes.js              |  40 ++-
 features/support/env.js                       | 110 ++++----
 features/support/exception_classes.js         | 132 +++++++++
 features/support/exceptions.js                |  15 +
 features/support/hash.js                      |  43 +++
 features/support/hooks.js                     |  73 ++---
 features/support/http.js                      |  12 +-
 features/support/launch.js                    |   5 +
 features/support/launch_classes.js            | 164 +++++++++++
 features/support/log.js                       |  90 ++++++
 features/support/route.js                     |   7 +-
 features/support/run.js                       |  70 ++---
 features/support/shared_steps.js              |  22 +-
 features/support/table_patch.js               |  11 +
 features/testbot/bad.feature                  |   4 +-
 features/testbot/bugs.feature                 |   5 +
 features/testbot/matching.feature             |   2 +-
 .../testbot/traffic_turn_penalties.feature    |   2 +-
 features/testbot/via.feature                  |   2 +-
 package.json                                  |   2 -
 profiles/rasterbot.lua                        |   6 +-
 profiles/rasterbotinterp.lua                  |   6 +-
 60 files changed, 1136 insertions(+), 863 deletions(-)
 delete mode 100644 features/lib/hash.js
 delete mode 100644 features/lib/osrm_loader.js
 delete mode 100644 features/lib/table_diff.js
 delete mode 100644 features/lib/try_connect.js
 delete mode 100644 features/lib/utils.js
 create mode 100644 features/step_definitions/hooks.js
 rename features/{lib/osm.js => support/build_osm.js} (96%)
 delete mode 100644 features/support/cache.js
 create mode 100644 features/support/config.js
 create mode 100644 features/support/exception_classes.js
 create mode 100644 features/support/exceptions.js
 create mode 100644 features/support/hash.js
 create mode 100644 features/support/launch.js
 create mode 100644 features/support/launch_classes.js
 create mode 100644 features/support/log.js
 create mode 100644 features/support/table_patch.js
 create mode 100644 features/testbot/bugs.feature

diff --git a/cucumber.js b/cucumber.js
index 0bf6f570f95..5a87dd7f1c2 100644
--- a/cucumber.js
+++ b/cucumber.js
@@ -1,8 +1,10 @@
 module.exports = {
-    default: '--strict --tags ~@stress --tags ~@todo --require features/support --require features/step_definitions',
-    verify: '--strict --tags ~@stress --tags ~@todo -f progress --require features/support --require features/step_definitions',
-    todo: '--strict --tags @todo --require features/support --require features/step_definitions',
-    all: '--strict --require features/support --require features/step_definitions'
+    default: '--require features --tags ~@stress --tags ~@todo',
+    verify: '--require features --tags ~@todo --tags ~@bug --tags ~@stress -f progress',
+    jenkins: '--require features --tags ~@todo --tags ~@bug --tags ~@stress --tags ~@options -f progress',
+    bugs: '--require features --tags @bug',
+    todo: '--require features --tags @todo',
+    all: '--require features'
 }
 
 
diff --git a/features/car/traffic_speeds.feature b/features/car/traffic_speeds.feature
index c26c6959926..c2ad0928806 100644
--- a/features/car/traffic_speeds.feature
+++ b/features/car/traffic_speeds.feature
@@ -25,7 +25,7 @@ Feature: Traffic - speeds
             | fb    | primary |
         Given the profile "testbot"
         Given the extract extra arguments "--generate-edge-lookup"
-        Given the contract extra arguments "--segment-speed-file {speeds_file}"
+        Given the contract extra arguments "--segment-speed-file speeds.csv"
         Given the speed file
         """
         1,2,0
@@ -69,7 +69,7 @@ Feature: Traffic - speeds
             | fb    | primary |
         Given the profile "testbot"
         Given the extract extra arguments "--generate-edge-lookup"
-        Given the contract extra arguments "--segment-speed-file {speeds_file}"
+        Given the contract extra arguments "--segment-speed-file speeds.csv"
         Given the speed file
         """
         1,2,0
@@ -112,6 +112,7 @@ Feature: Traffic - speeds
             | fb    | primary |
         Given the profile "testbot"
         Given the extract extra arguments "--generate-edge-lookup"
+        Given the contract extra arguments "--segment-speed-file speeds.csv"
         Given the speed file
         """
         1,2,-10
@@ -122,6 +123,6 @@ Feature: Traffic - speeds
         4,1,-5
         """
         And the data has been extracted
-        When I try to run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
+        When I run "osrm-contract --segment-speed-file speeds.csv {extracted_base}.osrm"
         And stderr should contain "malformed"
-        And it should exit with an error
+        And it should exit with code not 0
diff --git a/features/car/traffic_turn_penalties.feature b/features/car/traffic_turn_penalties.feature
index cb3907a5e50..09f5323b1d8 100644
--- a/features/car/traffic_turn_penalties.feature
+++ b/features/car/traffic_turn_penalties.feature
@@ -58,7 +58,7 @@ Feature: Traffic - turn penalties
             8,11,12,23
             1,4,5,-0.2
             """
-        And the contract extra arguments "--turn-penalty-file {penalties_file}"
+        And the contract extra arguments "--turn-penalty-file penalties.csv"
         When I route I should get
             | from | to | route                 | speed   | time      |
             | a    | h  | ad,dhk,dhk            | 63 km/h | 11.5s +-1 |
@@ -81,7 +81,7 @@ Feature: Traffic - turn penalties
                                                                               # double left - hdc penalty ever so slightly higher than imn; forces all the way around
 
     Scenario: Too-negative penalty clamps, but does not fail
-        Given the contract extra arguments "--turn-penalty-file {penalties_file}"
+        Given the contract extra arguments "--turn-penalty-file penalties.csv"
         And the profile "testbot"
         And the turn penalty file
             """
diff --git a/features/guidance/anticipate-lanes.feature b/features/guidance/anticipate-lanes.feature
index e235f2fbab4..f180b7a3343 100644
--- a/features/guidance/anticipate-lanes.feature
+++ b/features/guidance/anticipate-lanes.feature
@@ -422,7 +422,7 @@ Feature: Turn Lane Guidance
                | waypoints | route               | turns                                          | lanes                                                                                                                                                             |
                | a,e       | main,main,main,main | depart,use lane straight,continue right,arrive | ,left:false straight:false straight:false straight:false straight:true straight:true right:false,straight:false straight:false right:false right:true right:true, |
 
-    @anticipate @todo @2661
+    @anticipate @todo @bug @2661
     Scenario: Anticipate with lanes in roundabout: roundabouts as the unit of anticipation
         Given the node map
             |   |   | e |   |   |
@@ -667,7 +667,7 @@ Feature: Turn Lane Guidance
             | a,f       | abc,bdeh,feg,feg | depart,turn right,turn right,arrive | ,none:false none:false right:false right:true,left:false none:false none:false right:true, |
 
     @anticipate
-    Scenario: Triple Right keeping Left
+    Scenario: Tripple Right keeping Left
         Given the node map
             | a |   |   |   | b |   | i |
             |   |   |   |   |   |   |   |
diff --git a/features/guidance/turn-lanes.feature b/features/guidance/turn-lanes.feature
index ff6c750c3fa..e1d493b49c8 100644
--- a/features/guidance/turn-lanes.feature
+++ b/features/guidance/turn-lanes.feature
@@ -620,7 +620,7 @@ Feature: Turn Lane Guidance
             | a,d       | hwy,hwy       | depart,arrive                       | ,                                                                            |
             | a,e       | hwy,ramp,ramp | depart,off ramp slight right,arrive | ,straight:false straight:false straight;slight right:true slight right:true, |
 
-    @todo
+    @bug @todo
     Scenario: Turning Off Ramp
         Given the node map
             |   | a |   |
diff --git a/features/lib/hash.js b/features/lib/hash.js
deleted file mode 100644
index 57563e3da84..00000000000
--- a/features/lib/hash.js
+++ /dev/null
@@ -1,31 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const crypto = require('crypto');
-const d3 = require('d3-queue');
-
-module.exports =  {
-    hashOfFiles: (paths, cb) => {
-        let queue = d3.queue();
-        for (let i = 0; i < paths.length; ++i) {
-            queue.defer(fs.readFile, paths[i]);
-        }
-        queue.awaitAll((err, results) => {
-            if (err) return cb(err);
-            let checksum = crypto.createHash('md5');
-            for (let i = 0; i < results.length; ++i) {
-                checksum.update(results[i]);
-            }
-            cb(null, checksum.digest('hex'));
-        });
-    },
-
-     hashOfFile: (path, cb) => {
-        fs.readFile(path, (err, result) => {
-            if (err) return cb(err);
-            let checksum = crypto.createHash('md5');
-            checksum.update(result);
-            cb(null, checksum.digest('hex'));
-        });
-    }
-};
diff --git a/features/lib/osrm_loader.js b/features/lib/osrm_loader.js
deleted file mode 100644
index 28ec7ed2b42..00000000000
--- a/features/lib/osrm_loader.js
+++ /dev/null
@@ -1,169 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const util = require('util');
-const Timeout = require('node-timeout');
-const tryConnect = require('../lib/try_connect');
-const errorReason = require('./utils').errorReason;
-
-class OSRMBaseLoader{
-    constructor (scope) {
-        this.scope = scope;
-        this.child = null;
-    }
-
-    launch (callback) {
-        var limit = Timeout(this.scope.TIMEOUT, { err: new Error('*** Launching osrm-routed timed out.') });
-
-        var runLaunch = (cb) => {
-            this.osrmUp(() => { this.waitForConnection(cb); });
-        };
-
-        runLaunch(limit((e) => { if (e) callback(e); else callback(); }));
-    }
-
-    shutdown (callback) {
-        if (!this.osrmIsRunning()) return callback();
-
-        var limit = Timeout(this.scope.TIMEOUT, { err: new Error('*** Shutting down osrm-routed timed out.')});
-
-        this.osrmDown(limit(callback));
-    }
-
-    osrmIsRunning () {
-        return this.child && !this.child.killed;
-    }
-
-    osrmDown (callback) {
-        if (this.osrmIsRunning()) {
-            this.child.on('exit', (code, signal) => {callback();});
-            this.child.kill();
-        } else callback();
-    }
-
-    waitForConnection (callback) {
-        var retryCount = 0;
-        let retry = (err) => {
-          if (err) {
-            if (retryCount < 10) {
-              retryCount++;
-              setTimeout(() => { tryConnect(this.scope.OSRM_PORT, retry); }, 10);
-            } else {
-              callback(new Error("Could not connect to osrm-routed after ten retries."));
-            }
-          }
-          else
-          {
-            callback();
-          }
-        };
-
-        tryConnect(this.scope.OSRM_PORT, retry);
-    }
-};
-
-class OSRMDirectLoader extends OSRMBaseLoader {
-    constructor (scope) {
-        super(scope);
-    }
-
-    load (inputFile, callback) {
-        this.inputFile = inputFile;
-        this.shutdown(() => {
-            this.launch(callback);
-        });
-    }
-
-    osrmUp (callback) {
-        if (this.osrmIsRunning()) return callback(new Error("osrm-routed already running!"));
-
-        this.child = this.scope.runBin('osrm-routed', util.format("%s -p %d", this.inputFile, this.scope.OSRM_PORT), this.scope.environment, (err) => {
-          if (err) {
-              throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd));
-          }
-        });
-        callback();
-    }
-};
-
-class OSRMDatastoreLoader extends OSRMBaseLoader {
-    constructor (scope) {
-        super(scope);
-    }
-
-    load (inputFile, callback) {
-        this.inputFile = inputFile;
-
-        this.loadData((err) => {
-            if (err) return callback(err);
-            if (!this.osrmIsRunning()) this.launch(callback);
-            else {
-                this.scope.setupOutputLog(this.child, fs.createWriteStream(this.scope.scenarioLogFile, {'flags': 'a'}));
-                callback();
-            }
-        });
-    }
-
-    loadData (callback) {
-        this.scope.runBin('osrm-datastore', this.inputFile, this.scope.environment, (err) => {
-            if (err) return callback(new Error('*** osrm-datastore exited with ' + err.code + ': ' + err));
-            callback();
-        });
-    }
-
-    osrmUp (callback) {
-        if (this.osrmIsRunning()) return callback();
-
-        this.child = this.scope.runBin('osrm-routed', util.format('--shared-memory=1 -p %d', this.scope.OSRM_PORT), this.scope.environment, (err) => {
-            if (err) {
-                throw new Error(util.format('osrm-routed %s: %s', errorReason(err), err.cmd));
-            }
-        });
-
-        // we call the callback here, becuase we don't want to wait for the child process to finish
-        callback();
-    }
-};
-
-class OSRMLoader {
-    constructor (scope) {
-        this.scope = scope;
-        this.sharedLoader = new OSRMDatastoreLoader(this.scope);
-        this.directLoader = new OSRMDirectLoader(this.scope);
-        this.method = scope.DEFAULT_LOAD_METHOD;
-    }
-
-    load (inputFile, callback) {
-        if (this.method === 'datastore') {
-            this.directLoader.shutdown((err) => {
-              if (err) return callback(err);
-              this.loader = this.sharedLoader;
-              this.sharedLoader.load(inputFile, callback);
-            });
-        } else if (this.method === 'directly') {
-            this.sharedLoader.shutdown((err) => {
-              if (err) return callback(err);
-              this.loader = this.directLoader;
-              this.directLoader.load(inputFile, callback);
-            });
-        } else {
-            callback(new Error('*** Unknown load method ' + method));
-        }
-    }
-
-    setLoadMethod (method) {
-        this.method = method;
-    }
-
-    shutdown (callback) {
-        if (!this.loader) return callback();
-
-        this.loader.shutdown(callback);
-    }
-
-    up () {
-        return this.loader ? this.loader.osrmIsRunning() : false;
-    }
-};
-
-module.exports = OSRMLoader;
diff --git a/features/lib/table_diff.js b/features/lib/table_diff.js
deleted file mode 100644
index 4acbd23c139..00000000000
--- a/features/lib/table_diff.js
+++ /dev/null
@@ -1,54 +0,0 @@
-'use strict';
-
-var util = require('util');
-var path = require('path');
-var fs = require('fs');
-var chalk = require('chalk');
-
-var unescapeStr = (str) => str.replace(/\\\|/g, '\|').replace(/\\\\/g, '\\');
-
-module.exports = function (expected, actual) {
-    let headers = expected.raw()[0];
-    let expected_keys = expected.hashes();
-    let diff = [];
-    let hasErrors = false;
-
-    var good = 0, bad = 0;
-
-    expected_keys.forEach((row, i) => {
-        var rowError = false;
-
-        for (var j in row) {
-            if (unescapeStr(row[j]) != actual[i][j]) {
-                rowError = true;
-                hasErrors = true;
-                break;
-            }
-        }
-
-        if (rowError) {
-            bad++;
-            diff.push(Object.assign({}, row, {c_status: 'undefined'}));
-            diff.push(Object.assign({}, actual[i], {c_status: 'comment'}));
-        } else {
-            good++;
-            diff.push(row);
-        }
-    });
-
-    if (!hasErrors) return null;
-
-    var s = ['Tables were not identical:'];
-    s.push(headers.map(key => '    ' + key).join(' | '));
-    diff.forEach((row) => {
-        var rowString = '| ';
-        headers.forEach((header) => {
-            if (!row.c_status) rowString += chalk.green('    ' + row[header] + ' | ');
-            else if (row.c_status === 'undefined') rowString += chalk.yellow('(-) ' + row[header] + ' | ');
-            else rowString += chalk.red('(+) ' + row[header] + ' | ');
-        });
-        s.push(rowString);
-    });
-
-    return s.join('\n') + '\nTODO this is a temp workaround waiting for https://github.com/cucumber/cucumber-js/issues/534';
-};
diff --git a/features/lib/try_connect.js b/features/lib/try_connect.js
deleted file mode 100644
index 0461dddb8dd..00000000000
--- a/features/lib/try_connect.js
+++ /dev/null
@@ -1,13 +0,0 @@
-'use strict';
-
-const net = require('net');
-const Timeout = require('node-timeout');
-
-module.exports = function tryConnect(port, callback) {
-  net.connect({ port: port, host: '127.0.0.1' })
-    .on('connect', () => { callback(); })
-    .on('error', () => {
-        callback(new Error('Could not connect.'));
-    });
-}
-
diff --git a/features/lib/utils.js b/features/lib/utils.js
deleted file mode 100644
index 27b63af6cd1..00000000000
--- a/features/lib/utils.js
+++ /dev/null
@@ -1,17 +0,0 @@
-'use strict';
-
-const util = require('util');
-
-module.exports = {
-
-    ensureDecimal: (i) => {
-        if (parseInt(i) === i) return i.toFixed(1);
-        else return i;
-    },
-
-    errorReason: (err) => {
-        return err.signal ?
-            util.format('killed by signal %s', err.signal) :
-            util.format('exited with code %d', err.code);
-    }
-};
diff --git a/features/options/contract/datasources.feature b/features/options/contract/datasources.feature
index 19913885f88..bf8eed5d572 100644
--- a/features/options/contract/datasources.feature
+++ b/features/options/contract/datasources.feature
@@ -1,7 +1,8 @@
 @prepare @options @files
 Feature: osrm-contract command line options: datasources
 # expansions:
-# {processed_file} => path to .osrm file
+# {extracted_base} => path to current extracted input file
+# {profile} => path to current profile script
 
     Background:
         Given the profile "testbot"
@@ -23,6 +24,7 @@ Feature: osrm-contract command line options: datasources
         And the data has been extracted
 
     Scenario: osrm-contract - Passing base file
-        When I run "osrm-contract --segment-speed-file {speeds_file} {processed_file}"
-        Then datasource names should contain "lua profile,25_osrmcontract_passing_base_file_speeds"
-        And it should exit successfully
+        When I run "osrm-contract --segment-speed-file speeds.csv {extracted_base}.osrm"
+        Then stderr should be empty
+        And datasource names should contain "lua profile,speeds"
+        And it should exit with code 0
diff --git a/features/options/contract/files.feature b/features/options/contract/files.feature
index 46cf4fba01b..5e50e14f285 100644
--- a/features/options/contract/files.feature
+++ b/features/options/contract/files.feature
@@ -1,5 +1,9 @@
 @prepare @options @files
 Feature: osrm-contract command line options: files
+# expansions:
+# {extracted_base} => path to current extracted input file
+# {profile} => path to current profile script
+
     Background:
         Given the profile "testbot"
         And the node map
@@ -10,11 +14,12 @@ Feature: osrm-contract command line options: files
         And the data has been extracted
 
     Scenario: osrm-contract - Passing base file
-        When I run "osrm-contract {processed_file}"
-        Then it should exit successfully
+        When I run "osrm-contract {extracted_base}.osrm"
+        Then stderr should be empty
+        And it should exit with code 0
 
     Scenario: osrm-contract - Missing input file
-        When I try to run "osrm-contract over-the-rainbow.osrm"
+        When I run "osrm-contract over-the-rainbow.osrm"
         And stderr should contain "over-the-rainbow.osrm"
         And stderr should contain "not found"
-        And it should exit with an error
+        And it should exit with code 1
diff --git a/features/options/contract/help.feature b/features/options/contract/help.feature
index b4d81c557f3..411bc55da6b 100644
--- a/features/options/contract/help.feature
+++ b/features/options/contract/help.feature
@@ -2,7 +2,7 @@
 Feature: osrm-contract command line options: help
 
     Scenario: osrm-contract - Help should be shown when no options are passed
-        When I try to run "osrm-contract"
+        When I run "osrm-contract"
         Then stderr should be empty
         And stdout should contain "osrm-contract <input.osrm> [options]:"
         And stdout should contain "Options:"
@@ -13,7 +13,7 @@ Feature: osrm-contract command line options: help
         And stdout should contain "--core"
         And stdout should contain "--level-cache"
         And stdout should contain "--segment-speed-file"
-        And it should exit with an error
+        And it should exit with code 1
 
     Scenario: osrm-contract - Help, short
         When I run "osrm-contract -h"
@@ -27,7 +27,7 @@ Feature: osrm-contract command line options: help
         And stdout should contain "--core"
         And stdout should contain "--level-cache"
         And stdout should contain "--segment-speed-file"
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-contract - Help, long
         When I run "osrm-contract --help"
@@ -41,4 +41,4 @@ Feature: osrm-contract command line options: help
         And stdout should contain "--core"
         And stdout should contain "--level-cache"
         And stdout should contain "--segment-speed-file"
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/contract/invalid.feature b/features/options/contract/invalid.feature
index 127761ee3be..38ee3ace9ac 100644
--- a/features/options/contract/invalid.feature
+++ b/features/options/contract/invalid.feature
@@ -5,8 +5,8 @@ Feature: osrm-contract command line options: invalid options
         Given the profile "testbot"
 
     Scenario: osrm-contract - Non-existing option
-        When I try to run "osrm-contract --fly-me-to-the-moon"
+        When I run "osrm-contract --fly-me-to-the-moon"
         Then stdout should be empty
         And stderr should contain "option"
         And stderr should contain "fly-me-to-the-moon"
-        And it should exit with an error
+        And it should exit with code 1
diff --git a/features/options/contract/version.feature b/features/options/contract/version.feature
index f361bb1e531..be99bbed150 100644
--- a/features/options/contract/version.feature
+++ b/features/options/contract/version.feature
@@ -12,11 +12,11 @@ Feature: osrm-contract command line options: version
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-contract - Version, long
         When I run "osrm-contract --version"
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/extract/files.feature b/features/options/extract/files.feature
index c4e14a278f3..aceab19f74a 100644
--- a/features/options/extract/files.feature
+++ b/features/options/extract/files.feature
@@ -14,15 +14,17 @@ Feature: osrm-extract command line options: files
         And the data has been saved to disk
 
     Scenario: osrm-extract - Passing base file
-        When I run "osrm-extract {osm_file} --profile {profile_file}"
-        Then it should exit successfully
+        When I run "osrm-extract {osm_base}.osm --profile {profile}"
+        Then stderr should be empty
+        And it should exit with code 0
 
     Scenario: osrm-extract - Order of options should not matter
-        When I run "osrm-extract --profile {profile_file} {osm_file}"
-        Then it should exit successfully
+        When I run "osrm-extract --profile {profile} {osm_base}.osm"
+        Then stderr should be empty
+        And it should exit with code 0
 
     Scenario: osrm-extract - Missing input file
-        When I try to run "osrm-extract over-the-rainbow.osrm --profile {profile_file}"
+        When I run "osrm-extract over-the-rainbow.osrm --profile {profile}"
         And stderr should contain "over-the-rainbow.osrm"
         And stderr should contain "not found"
-        And it should exit with an error
+        And it should exit with code 1
diff --git a/features/options/extract/help.feature b/features/options/extract/help.feature
index 0d400edbafe..cdf1eb9a36a 100644
--- a/features/options/extract/help.feature
+++ b/features/options/extract/help.feature
@@ -16,7 +16,7 @@ Feature: osrm-extract command line options: help
         And stdout should contain "--threads"
         And stdout should contain "--generate-edge-lookup"
         And stdout should contain "--small-component-size"
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-extract - Help, short
         When I run "osrm-extract -h"
@@ -30,7 +30,7 @@ Feature: osrm-extract command line options: help
         And stdout should contain "--threads"
         And stdout should contain "--generate-edge-lookup"
         And stdout should contain "--small-component-size"
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-extract - Help, long
         When I run "osrm-extract --help"
@@ -44,4 +44,4 @@ Feature: osrm-extract command line options: help
         And stdout should contain "--threads"
         And stdout should contain "--generate-edge-lookup"
         And stdout should contain "--small-component-size"
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/extract/invalid.feature b/features/options/extract/invalid.feature
index 936f456fb43..169e53caa9c 100644
--- a/features/options/extract/invalid.feature
+++ b/features/options/extract/invalid.feature
@@ -5,8 +5,8 @@ Feature: osrm-extract command line options: invalid options
         Given the profile "testbot"
 
     Scenario: osrm-extract - Non-existing option
-        When I try to run "osrm-extract --fly-me-to-the-moon"
+        When I run "osrm-extract --fly-me-to-the-moon"
         Then stdout should be empty
         And stderr should contain "option"
         And stderr should contain "fly-me-to-the-moon"
-        And it should exit with an error
+        And it should exit with code 1
diff --git a/features/options/extract/version.feature b/features/options/extract/version.feature
index 77ee46cd87c..0dd5f65886b 100644
--- a/features/options/extract/version.feature
+++ b/features/options/extract/version.feature
@@ -12,11 +12,11 @@ Feature: osrm-extract command line options: version
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-extract - Version, long
         When I run "osrm-extract --version"
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/routed/files.feature b/features/options/routed/files.feature
index b28c8b11a27..59ce7c21336 100644
--- a/features/options/routed/files.feature
+++ b/features/options/routed/files.feature
@@ -29,4 +29,4 @@ Feature: osrm-routed command line options: files
         And stdout should contain /^\[info\] loaded plugin: viaroute/
         And stdout should contain /^\[info\] trial run/
         And stdout should contain /^\[info\] shutdown completed/
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/routed/help.feature b/features/options/routed/help.feature
index e8c6430f6dd..8f64bd96753 100644
--- a/features/options/routed/help.feature
+++ b/features/options/routed/help.feature
@@ -21,7 +21,7 @@ Feature: osrm-routed command line options: help
         And stdout should contain "--max-trip-size"
         And stdout should contain "--max-table-size"
         And stdout should contain "--max-matching-size"
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-routed - Help, short
         When I run "osrm-routed -h"
@@ -40,7 +40,7 @@ Feature: osrm-routed command line options: help
         And stdout should contain "--max-trip-size"
         And stdout should contain "--max-table-size"
         And stdout should contain "--max-matching-size"
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-routed - Help, long
         When I run "osrm-routed --help"
@@ -59,4 +59,4 @@ Feature: osrm-routed command line options: help
         And stdout should contain "--max-table-size"
         And stdout should contain "--max-table-size"
         And stdout should contain "--max-matching-size"
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/options/routed/invalid.feature b/features/options/routed/invalid.feature
index 78d28a06411..9c84357840c 100644
--- a/features/options/routed/invalid.feature
+++ b/features/options/routed/invalid.feature
@@ -5,14 +5,14 @@ Feature: osrm-routed command line options: invalid options
         Given the profile "testbot"
 
     Scenario: osrm-routed - Non-existing option
-        When I try to run "osrm-routed --fly-me-to-the-moon"
+        When I run "osrm-routed --fly-me-to-the-moon"
         Then stdout should be empty
         And stderr should contain "unrecognised"
         And stderr should contain "fly-me-to-the-moon"
-        And it should exit with an error
+        And it should exit with code 1
 
     Scenario: osrm-routed - Missing file
-        When I try to run "osrm-routed over-the-rainbow.osrm"
+        When I run "osrm-routed over-the-rainbow.osrm"
         Then stderr should contain "over-the-rainbow.osrm"
         And stderr should contain "not found"
-        And it should exit with an error
+        And it should exit with code 1
diff --git a/features/options/routed/version.feature b/features/options/routed/version.feature
index 0a3cad55e0a..b544e36e6e5 100644
--- a/features/options/routed/version.feature
+++ b/features/options/routed/version.feature
@@ -12,11 +12,11 @@ Feature: osrm-routed command line options: version
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
 
     Scenario: osrm-routed - Version, long
         When I run "osrm-routed --version"
         Then stderr should be empty
         And stdout should contain 1 line
         And stdout should contain /(v\d{1,2}\.\d{1,2}\.\d{1,2}|\w*-\d+-\w+)/
-        And it should exit successfully
+        And it should exit with code 0
diff --git a/features/raster/extract.feature b/features/raster/extract.feature
index 86a7537163f..9ca0635d87e 100644
--- a/features/raster/extract.feature
+++ b/features/raster/extract.feature
@@ -1,5 +1,9 @@
 @raster @extract
 Feature: osrm-extract with a profile containing raster source
+# expansions:
+# {osm_base} => path to current input file
+# {profile} => path to current profile script
+
     Scenario: osrm-extract on a valid profile
         Given the profile "rasterbot"
         And the node map
@@ -7,15 +11,8 @@ Feature: osrm-extract with a profile containing raster source
         And the ways
             | nodes |
             | ab    |
-        And the raster source
-            """
-            0  0  0   0
-            0  0  0   250
-            0  0  250 500
-            0  0  0   250
-            0  0  0   0
-            """
         And the data has been saved to disk
-        When I run "osrm-extract {osm_file} -p {profile_file}"
-        Then stdout should contain "source loader"
-        And it should exit successfully
+        When I run "osrm-extract {osm_base}.osm -p {profile}"
+        Then stderr should be empty
+        And stdout should contain "source loader"
+        And it should exit with code 0
diff --git a/features/raster/weights.feature b/features/raster/weights.feature
index 1c03bdc1ba8..ae782a72281 100644
--- a/features/raster/weights.feature
+++ b/features/raster/weights.feature
@@ -32,8 +32,8 @@ Feature: Raster - weights
 
     Scenario: Weighting not based on raster sources
         Given the profile "testbot"
-        When I run "osrm-extract {osm_file} -p {profile_file}"
-        And I run "osrm-contract {processed_file}"
+        When I run "osrm-extract {osm_base}.osm -p {profile}"
+        And I run "osrm-contract {osm_base}.osm"
         And I route I should get
             | from | to | route    | speed   |
             | a    | b  | ab,ab    | 36 km/h |
@@ -44,9 +44,9 @@ Feature: Raster - weights
 
     Scenario: Weighting based on raster sources
         Given the profile "rasterbot"
-        When I run "osrm-extract {osm_file} -p {profile_file}"
+        When I run "osrm-extract {osm_base}.osm -p {profile}"
         Then stdout should contain "evaluating segment"
-        And I run "osrm-contract {processed_file}"
+        And I run "osrm-contract {osm_base}.osm"
         And I route I should get
             | from | to | route    | speed   |
             | a    | b  | ab,ab    | 8 km/h  |
@@ -62,9 +62,9 @@ Feature: Raster - weights
 
     Scenario: Weighting based on raster sources
         Given the profile "rasterbotinterp"
-        When I run "osrm-extract {osm_file} -p {profile_file}"
+        When I run "osrm-extract {osm_base}.osm -p {profile}"
         Then stdout should contain "evaluating segment"
-        And I run "osrm-contract {processed_file}"
+        And I run "osrm-contract {osm_base}.osm"
         And I route I should get
             | from | to | route    | speed   |
             | a    | b  | ab,ab    | 8 km/h  |
diff --git a/features/step_definitions/data.js b/features/step_definitions/data.js
index dbb3882d1d0..58a3fc5ff1f 100644
--- a/features/step_definitions/data.js
+++ b/features/step_definitions/data.js
@@ -2,23 +2,19 @@ var util = require('util');
 var path = require('path');
 var fs = require('fs');
 var d3 = require('d3-queue');
-var OSM = require('../lib/osm');
+var OSM = require('../support/build_osm');
 
 module.exports = function () {
     this.Given(/^the profile "([^"]*)"$/, (profile, callback) => {
-        this.profile = profile;
-        this.profileFile = path.join(this.PROFILES_PATH, this.profile + '.lua');
-        callback();
+        this.setProfile(profile, callback);
     });
 
     this.Given(/^the extract extra arguments "(.*?)"$/, (args, callback) => {
-        this.extractArgs = this.expandOptions(args);
-        callback();
+        this.setExtractArgs(args, callback);
     });
 
     this.Given(/^the contract extra arguments "(.*?)"$/, (args, callback) => {
-        this.contractArgs = this.expandOptions(args);
-        callback();
+        this.setContractArgs(args, callback);
     });
 
     this.Given(/^a grid size of ([0-9.]+) meters$/, (meters, callback) => {
@@ -232,46 +228,58 @@ module.exports = function () {
     });
 
     this.Given(/^the raster source$/, (data, callback) => {
-        // TODO: Don't overwrite if it exists
-        fs.writeFile(this.rasterCacheFile, data, callback);
-        // we need this to pass it to the profiles
-        this.environment = Object.assign({OSRM_RASTER_SOURCE: this.rasterCacheFile}, this.environment);
+        this.updateFingerprintExtract(data);
+        fs.writeFile(path.resolve(this.TEST_FOLDER, 'rastersource.asc'), data, callback);
     });
 
     this.Given(/^the speed file$/, (data, callback) => {
-        // TODO: Don't overwrite if it exists
-        fs.writeFile(this.speedsCacheFile, data, callback);
+        this.updateFingerprintContract(data);
+        fs.writeFile(path.resolve(this.TEST_FOLDER, 'speeds.csv'), data, callback);
     });
 
     this.Given(/^the turn penalty file$/, (data, callback) => {
-        // TODO: Don't overwrite if it exists
-        fs.writeFile(this.penaltiesCacheFile, data, callback);
+        this.updateFingerprintContract(data);
+        fs.writeFile(path.resolve(this.TEST_FOLDER, 'penalties.csv'), data, callback);
     });
 
     this.Given(/^the data has been saved to disk$/, (callback) => {
-        this.reprocess(callback);
+        try {
+            this.reprocess(callback);
+        } catch(e) {
+            this.processError = e;
+            callback(e);
+        }
     });
 
     this.Given(/^the data has been extracted$/, (callback) => {
-        this.reprocess(callback);
+        this.osmData.populate(() => {
+            this.writeAndExtract((err) => {
+                if (err) this.processError = err;
+                callback();
+            });
+        });
     });
 
     this.Given(/^the data has been contracted$/, (callback) => {
-        this.reprocess(callback);
+        this.reprocess((err) => {
+            if (err) this.processError = err;
+            callback();
+        });
     });
 
     this.Given(/^osrm\-routed is stopped$/, (callback) => {
-        this.OSRMLoader.shutdown(callback);
+        this.OSRMLoader.shutdown((err) => {
+            if (err) this.processError = err;
+            callback();
+        });
     });
 
-    this.Given(/^data is loaded directly/, (callback) => {
-        this.osrmLoader.setLoadMethod('directly');
-        callback();
+    this.Given(/^data is loaded directly/, () => {
+        this.loadMethod = 'directly';
     });
 
-    this.Given(/^data is loaded with datastore$/, (callback) => {
-        this.osrmLoader.setLoadMethod('datastore');
-        callback();
+    this.Given(/^data is loaded with datastore$/, () => {
+        this.loadMethod = 'datastore';
     });
 
     this.Given(/^the HTTP method "([^"]*)"$/, (method, callback) => {
diff --git a/features/step_definitions/distance_matrix.js b/features/step_definitions/distance_matrix.js
index 9c2bc0adda0..f032f08e92e 100644
--- a/features/step_definitions/distance_matrix.js
+++ b/features/step_definitions/distance_matrix.js
@@ -53,6 +53,8 @@ module.exports = function () {
                 });
 
                 var testRow = (row, ri, cb) => {
+                    var ok = true;
+
                     for (var k in result[ri]) {
                         if (this.FuzzyMatch.match(result[ri][k], row[k])) {
                             result[ri][k] = row[k];
@@ -60,9 +62,15 @@ module.exports = function () {
                             result[ri][k] = '';
                         } else {
                             result[ri][k] = result[ri][k].toString();
+                            ok = false;
                         }
                     }
 
+                    if (!ok) {
+                        var failed = { attempt: 'distance_matrix', query: this.query, response: response };
+                        this.logFail(row, result[ri], [failed]);
+                    }
+
                     result[ri][''] = row[''];
                     cb(null, result[ri]);
                 };
diff --git a/features/step_definitions/hooks.js b/features/step_definitions/hooks.js
new file mode 100644
index 00000000000..d6ed251b46e
--- /dev/null
+++ b/features/step_definitions/hooks.js
@@ -0,0 +1,18 @@
+var util = require('util');
+
+module.exports = function () {
+    this.Before((scenario, callback) => {
+        this.scenarioTitle = scenario.getName();
+
+        this.loadMethod = this.DEFAULT_LOAD_METHOD;
+        this.queryParams = {};
+        var d = new Date();
+        this.scenarioTime = util.format('%d-%d-%dT%s:%s:%sZ', d.getFullYear(), d.getMonth()+1, d.getDate(), d.getHours(), d.getMinutes(), d.getSeconds());
+        this.resetData();
+        this.hasLoggedPreprocessInfo = false;
+        this.hasLoggedScenarioInfo = false;
+        this.setGridSize(this.DEFAULT_GRID_SIZE);
+        this.setOrigin(this.DEFAULT_ORIGIN);
+        callback();
+    });
+};
diff --git a/features/step_definitions/matching.js b/features/step_definitions/matching.js
index 41cd0ea37ab..e3dc4e6e3a1 100644
--- a/features/step_definitions/matching.js
+++ b/features/step_definitions/matching.js
@@ -157,6 +157,7 @@ module.exports = function () {
                         } else {
                             got.matchings = encodedResult;
                             row.matchings = extendedTarget;
+                            this.logFail(row, got, { matching: { query: this.query, response: res } });
                         }
 
                         cb(null, got);
diff --git a/features/step_definitions/nearest.js b/features/step_definitions/nearest.js
index 450dce20a7b..919cb857dc1 100644
--- a/features/step_definitions/nearest.js
+++ b/features/step_definitions/nearest.js
@@ -22,16 +22,24 @@ module.exports = function () {
 
                         var got = { in: row.in, out: row.out };
 
+                        var ok = true;
+
                         Object.keys(row).forEach((key) => {
                             if (key === 'out') {
                                 if (this.FuzzyMatch.matchLocation(coord, outNode)) {
                                     got[key] = row[key];
                                 } else {
                                     row[key] = util.format('%s [%d,%d]', row[key], outNode.lat, outNode.lon);
+                                    ok = false;
                                 }
                             }
                         });
 
+                        if (!ok) {
+                            var failed = { attempt: 'nearest', query: this.query, response: response };
+                            this.logFail(row, got, [failed]);
+                        }
+
                         cb(null, got);
                     }
                     else {
diff --git a/features/step_definitions/options.js b/features/step_definitions/options.js
index d2177bffd44..5dc6929925e 100644
--- a/features/step_definitions/options.js
+++ b/features/step_definitions/options.js
@@ -2,58 +2,36 @@ var assert = require('assert');
 var fs = require('fs');
 
 module.exports = function () {
-    this.resetOptionsOutput = () => {
-        this.stdout = null;
-        this.stderr = null;
-        this.exitCode = null;
-        this.termSignal = null;
-    };
-
-    this.runAndSafeOutput = (binary, options, callback) => {
-        this.runBin(binary, this.expandOptions(options), this.environment, (err, stdout, stderr) => {
-            this.stdout = stdout;
-            this.stderr = stderr;
-            this.exitCode = err && err.code || 0;
-            this.termSignal = err && err.signal || '';
-            callback(err);
-        });
-    };
-
     this.When(/^I run "osrm\-routed\s?(.*?)"$/, { timeout: this.TIMEOUT }, (options, callback) => {
-        this.runAndSafeOutput('osrm-routed', options, callback);
+        this.runBin('osrm-routed', options, () => {
+            callback();
+        });
     });
 
     this.When(/^I run "osrm\-extract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-extract', options, callback);
+        this.runBin('osrm-extract', options, () => {
+            callback();
+        });
     });
 
     this.When(/^I run "osrm\-contract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-contract', options, callback);
-    });
-
-    this.When(/^I try to run "osrm\-routed\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-routed', options, () => { callback(); });
-    });
-
-    this.When(/^I try to run "osrm\-extract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-extract', options, () => { callback(); });
-    });
-
-    this.When(/^I try to run "osrm\-contract\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-contract', options, () => { callback(); });
+        this.runBin('osrm-contract', options, () => {
+            callback();
+        });
     });
 
     this.When(/^I run "osrm\-datastore\s?(.*?)"$/, (options, callback) => {
-        this.runAndSafeOutput('osrm-datastore', options, callback);
+        this.runBin('osrm-datastore', options, () => {
+            callback();
+        });
     });
 
-    this.Then(/^it should exit successfully$/, () => {
-        assert.equal(this.exitCode, 0);
-        assert.equal(this.termSignal, '');
+    this.Then(/^it should exit with code (\d+)$/, (code) => {
+        assert.equal(this.exitCode, parseInt(code));
     });
 
-    this.Then(/^it should exit with an error$/, () => {
-        assert.ok(this.exitCode !== 0 || this.termSignal);
+    this.Then(/^it should exit with code not (\d+)$/, (code) => {
+        assert.notEqual(this.exitCode, parseInt(code));
     });
 
     this.Then(/^stdout should contain "(.*?)"$/, (str) => {
@@ -87,7 +65,7 @@ module.exports = function () {
     });
 
     this.Then(/^datasource names should contain "(.+)"$/, (expectedData) => {
-        var actualData = fs.readFileSync(this.processedCacheFile + '.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(',');
+        var actualData = fs.readFileSync(this.osmData.extractedFile + '.osrm.datasource_names', {encoding:'UTF-8'}).trim().split('\n').join(',');
         assert.equal(actualData, expectedData);
     });
 
diff --git a/features/step_definitions/requests.js b/features/step_definitions/requests.js
index 36eef830bc9..cb118845744 100644
--- a/features/step_definitions/requests.js
+++ b/features/step_definitions/requests.js
@@ -51,7 +51,7 @@ module.exports = function () {
     });
 
     this.Then(/^"([^"]*)" should return code (\d+)$/, (binary, code) => {
-        assert.ok(this.processError instanceof Error);
+        assert.ok(this.processError instanceof this.OSRMError);
         assert.equal(this.processError.process, binary);
         assert.equal(parseInt(this.processError.code), parseInt(code));
     });
diff --git a/features/step_definitions/routability.js b/features/step_definitions/routability.js
index ea2a45e656b..c4190c547c5 100644
--- a/features/step_definitions/routability.js
+++ b/features/step_definitions/routability.js
@@ -13,7 +13,7 @@ module.exports = function () {
             }
 
             this.reprocessAndLoadData((e) => {
-                if (e) return callback(e);
+                if (e) callback(e);
                 var testRow = (row, i, cb) => {
                     var outputRow = row;
 
@@ -41,6 +41,10 @@ module.exports = function () {
                             }
                         });
 
+                        if (outputRow != row) {
+                            this.logFail(row, outputRow, result);
+                        }
+
                         cb(null, outputRow);
                     });
                 };
@@ -112,7 +116,7 @@ module.exports = function () {
                     sq.defer(parseRes, key);
                 });
 
-                sq.awaitAll((err) => { cb(err, result); });
+                sq.awaitAll(() => { cb(null, result); });
             });
     };
 };
diff --git a/features/step_definitions/trip.js b/features/step_definitions/trip.js
index 57ce0795f13..fe4ff892bb2 100644
--- a/features/step_definitions/trip.js
+++ b/features/step_definitions/trip.js
@@ -85,14 +85,23 @@ module.exports = function () {
                     } else {
                         got.trips = encodedResult;
                         got.trips = extendedTarget;
+                        this.logFail(row, got, { trip: { query: this.query, response: res }});
                     }
 
+                    ok = true;
+
                     for (var key in row) {
                         if (this.FuzzyMatch.match(got[key], row[key])) {
                             got[key] = row[key];
+                        } else {
+                            ok = false;
                         }
                     }
 
+                    if (!ok) {
+                        this.logFail(row, got, { trip: { query: this.query, response: res }});
+                    }
+
                     cb(null, got);
                 };
 
diff --git a/features/lib/osm.js b/features/support/build_osm.js
similarity index 96%
rename from features/lib/osm.js
rename to features/support/build_osm.js
index 5387b7f0110..7fe6874aeea 100644
--- a/features/lib/osm.js
+++ b/features/support/build_osm.js
@@ -1,7 +1,11 @@
 'use strict';
 
-const builder = require('xmlbuilder');
-const ensureDecimal = require('./utils').ensureDecimal;
+var builder = require('xmlbuilder');
+
+var ensureDecimal = (i) => {
+    if (parseInt(i) === i) return i.toFixed(1);
+    else return i;
+};
 
 class DB {
     constructor () {
diff --git a/features/support/cache.js b/features/support/cache.js
deleted file mode 100644
index e388f597613..00000000000
--- a/features/support/cache.js
+++ /dev/null
@@ -1,184 +0,0 @@
-'use strict';
-
-const d3 = require('d3-queue');
-const fs = require('fs');
-const util = require('util');
-const path = require('path');
-const mkdirp = require('mkdirp');
-const hash = require('../lib/hash');
-const rimraf = require('rimraf');
-
-module.exports = function() {
-    this.initializeCache = (callback) => {
-        this.getOSRMHash((err, osrmHash) => {
-            if (err) return callback(err);
-            this.osrmHash = osrmHash;
-            callback();
-        });
-    };
-
-    // computes all paths for every feature
-    this.setupFeatures = (features, callback) => {
-        this.featureIDs = {};
-        this.featureCacheDirectories = {};
-        this.featureProcessedCacheDirectories = {};
-        let queue = d3.queue();
-
-        function initializeFeature(feature, callback) {
-            let uri = feature.getUri();
-
-            // setup cache for feature data
-            hash.hashOfFile(uri, (err, hash) => {
-                if (err) return callback(err);
-
-                // shorten uri to be realtive to 'features/'
-                let featurePath = path.relative(path.resolve('./features'), uri);
-                // bicycle/bollards/{HASH}/
-                let featureID = path.join(featurePath, hash);
-                let featureCacheDirectory = this.getFeatureCacheDirectory(featureID);
-                let featureProcessedCacheDirectory = this.getFeatureProcessedCacheDirectory(featureCacheDirectory, this.osrmHash);
-                this.featureIDs[uri] = featureID;
-                this.featureCacheDirectories[uri] = featureCacheDirectory;
-                this.featureProcessedCacheDirectories[uri] = featureProcessedCacheDirectory;
-
-                d3.queue(1)
-                  .defer(mkdirp, featureProcessedCacheDirectory)
-                  .defer(this.cleanupFeatureCache.bind(this), featureCacheDirectory, hash)
-                  .defer(this.cleanupProcessedFeatureCache.bind(this), featureProcessedCacheDirectory, this.osrmHash)
-                  .awaitAll(callback);
-            });
-        }
-
-        for (let i = 0; i < features.length; ++i) {
-            queue.defer(initializeFeature.bind(this), features[i]);
-        }
-        queue.awaitAll(callback);
-    };
-
-    this.cleanupProcessedFeatureCache = (directory, osrmHash, callback) => {
-        let parentPath = path.resolve(path.join(directory, '..'));
-        fs.readdir(parentPath, (err, files) => {
-            let q = d3.queue();
-            function runStats(path, callback) {
-                fs.stat(path, (err, stat) => {
-                    if (err) return callback(err);
-                    callback(null, {file: path, stat: stat});
-                });
-            }
-            files.map(f => { q.defer(runStats, path.join(parentPath, f)); });
-            q.awaitAll((err, results) => {
-                if (err) return callback(err);
-                let q = d3.queue();
-                results.forEach(r => {
-                    if (r.stat.isDirectory() && r.file.search(osrmHash) < 0) {
-                        q.defer(rimraf, r.file);
-                    }
-                });
-                q.awaitAll(callback);
-            });
-        });
-    };
-
-    this.cleanupFeatureCache = (directory, featureHash, callback) => {
-        let parentPath = path.resolve(path.join(directory, '..'));
-        fs.readdir(parentPath, (err, files) => {
-            let q = d3.queue();
-            files.filter(name => { return name !== featureHash;})
-                 .map((f) => { q.defer(rimraf, path.join(parentPath, f)); });
-            q.awaitAll(callback);
-        });
-    };
-
-    this.setupFeatureCache = (feature) => {
-        let uri = feature.getUri();
-        this.featureID = this.featureIDs[uri];
-        this.featureCacheDirectory = this.featureCacheDirectories[uri];
-        this.featureProcessedCacheDirectory = this.featureProcessedCacheDirectories[uri];
-    };
-
-    this.setupScenarioCache = (scenarioID) => {
-        this.scenarioCacheFile = this.getScenarioCacheFile(this.featureCacheDirectory, scenarioID);
-        this.processedCacheFile = this.getProcessedCacheFile(this.featureProcessedCacheDirectory, scenarioID);
-        this.inputCacheFile = this.getInputCacheFile(this.featureProcessedCacheDirectory, scenarioID);
-        this.rasterCacheFile = this.getRasterCacheFile(this.featureProcessedCacheDirectory, scenarioID);
-        this.speedsCacheFile = this.getSpeedsCacheFile(this.featureProcessedCacheDirectory, scenarioID);
-        this.penaltiesCacheFile = this.getPenaltiesCacheFile(this.featureProcessedCacheDirectory, scenarioID);
-    };
-
-    // returns a hash of all OSRM code side dependencies
-    this.getOSRMHash = (callback) => {
-        let dependencies = [
-            this.OSRM_EXTRACT_PATH,
-            this.OSRM_CONTRACT_PATH,
-            this.LIB_OSRM_EXTRACT_PATH,
-            this.LIB_OSRM_CONTRACT_PATH
-        ];
-
-        var addLuaFiles = (directory, callback) => {
-            fs.readdir(path.normalize(directory), (err, files) => {
-                if (err) return callback(err);
-
-                var luaFiles = files.filter(f => !!f.match(/\.lua$/)).map(f => path.normalize(directory + '/' + f));
-                Array.prototype.push.apply(dependencies, luaFiles);
-
-                callback();
-            });
-        };
-
-        // Note: we need a serialized queue here to ensure that the order of the files
-        // passed is stable. Otherwise the hash will not be stable
-        d3.queue(1)
-            .defer(addLuaFiles, this.PROFILES_PATH)
-            .defer(addLuaFiles, this.PROFILES_PATH + '/lib')
-            .awaitAll(hash.hashOfFiles.bind(hash, dependencies, callback));
-    };
-
-    // test/cache/bicycle/bollards/{HASH}/
-    this.getFeatureCacheDirectory = (featureID) => {
-        return path.join(this.CACHE_PATH, featureID);
-    };
-
-    // converts the scenario titles in file prefixes
-    this.getScenarioID = (scenario) => {
-        let name = scenario.getName().toLowerCase().replace(/[\/\-'=,\(\)]/g, '').replace(/\s/g, '_').replace(/__/g, '_').replace(/\.\./g, '.');
-        return util.format('%d_%s', scenario.getLine(), name);
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{scenario}_raster.asc
-    this.getRasterCacheFile = (featureCacheDirectory, scenarioID) => {
-        return path.join(featureCacheDirectory, scenarioID) + '_raster.asc';
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{scenario}_speeds.csv
-    this.getSpeedsCacheFile = (featureCacheDirectory, scenarioID) => {
-        return path.join(featureCacheDirectory, scenarioID) + '_speeds.csv';
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{scenario}_penalties.csv
-    this.getPenaltiesCacheFile = (featureCacheDirectory, scenarioID) => {
-        return path.join(featureCacheDirectory, scenarioID) + '_penalties.csv';
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{scenario}.osm
-    this.getScenarioCacheFile = (featureCacheDirectory, scenarioID) => {
-        return path.join(featureCacheDirectory, scenarioID) + '.osm';
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/
-    this.getFeatureProcessedCacheDirectory = (featureCacheDirectory, osrmHash) => {
-        return path.join(featureCacheDirectory, osrmHash);
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osrm
-    this.getProcessedCacheFile = (featureProcessedCacheDirectory, scenarioID) => {
-        return path.join(featureProcessedCacheDirectory, scenarioID) + '.osrm';
-    };
-
-    // test/cache/{feature_path}/{feature_hash}/{osrm_hash}/{scenario}.osm
-    this.getInputCacheFile = (featureProcessedCacheDirectory, scenarioID) => {
-        return path.join(featureProcessedCacheDirectory, scenarioID) + '.osm';
-    };
-
-
-    return this;
-};
diff --git a/features/support/config.js b/features/support/config.js
new file mode 100644
index 00000000000..769755f4f3b
--- /dev/null
+++ b/features/support/config.js
@@ -0,0 +1,127 @@
+var fs = require('fs');
+var path = require('path');
+var util = require('util');
+var d3 = require('d3-queue');
+var OSM = require('./build_osm');
+var classes = require('./data_classes');
+
+module.exports = function () {
+    this.initializeOptions = (callback) => {
+        this.profile = this.profile || this.DEFAULT_SPEEDPROFILE;
+
+        this.OSMDB = this.OSMDB || new OSM.DB();
+
+        this.nameNodeHash = this.nameNodeHash || {};
+
+        this.locationHash = this.locationHash || {};
+
+        this.nameWayHash = this.nameWayHash || {};
+
+        this.osmData = new classes.osmData(this);
+
+        this.OSRMLoader = this._OSRMLoader();
+
+        this.PREPROCESS_LOG_FILE = path.resolve(this.TEST_FOLDER, 'preprocessing.log');
+
+        this.LOG_FILE = path.resolve(this.TEST_FOLDER, 'fail.log');
+
+        this.HOST = 'http://127.0.0.1:' + this.OSRM_PORT;
+
+        this.DESTINATION_REACHED = 15;  // OSRM instruction code
+
+        this.shortcutsHash = this.shortcutsHash || {};
+
+        var hashLuaLib = (cb) => {
+            fs.readdir(path.normalize(this.PROFILES_PATH + '/lib/'), (err, files) => {
+                if (err) cb(err);
+                var luaFiles = files.filter(f => !!f.match(/\.lua$/)).map(f => path.normalize(this.PROFILES_PATH + '/lib/' + f));
+                this.hashOfFiles(luaFiles, hash => {
+                    this.luaLibHash = hash;
+                    cb();
+                });
+            });
+        };
+
+        var hashProfile = (cb) => {
+            this.hashProfile((hash) => {
+                this.profileHash = hash;
+                cb();
+            });
+        };
+
+        var hashExtract = (cb) => {
+            var files = [ util.format('%s/osrm-extract%s', this.BIN_PATH, this.EXE),
+                          util.format('%s/libosrm_extract%s', this.BIN_PATH, this.LIB) ];
+            this.hashOfFiles(files, (hash) => {
+                this.binExtractHash = hash;
+                cb();
+            });
+        };
+
+        var hashContract = (cb) => {
+            var files = [ util.format('%s/osrm-contract%s', this.BIN_PATH, this.EXE),
+                          util.format('%s/libosrm_contract%s', this.BIN_PATH, this.LIB) ];
+            this.hashOfFiles(files, (hash) => {
+                this.binContractHash = hash;
+                cb();
+            });
+        };
+
+        var hashRouted = (cb) => {
+            var files = [ util.format('%s/osrm-routed%s', this.BIN_PATH, this.EXE),
+                          util.format('%s/libosrm%s', this.BIN_PATH, this.LIB) ];
+            this.hashOfFiles(files, (hash) => {
+                this.binRoutedHash = hash;
+                this.fingerprintRoute = this.hashString(this.binRoutedHash);
+                cb();
+            });
+        };
+
+        d3.queue()
+            .defer(hashLuaLib)
+            .defer(hashProfile)
+            .defer(hashExtract)
+            .defer(hashContract)
+            .defer(hashRouted)
+            .awaitAll(() => {
+                this.AfterConfiguration(() => {
+                    callback();
+                });
+            });
+    };
+
+    this.updateFingerprintExtract = (str) => {
+        this.fingerprintExtract = this.hashString([this.fingerprintExtract, str].join('-'));
+    };
+
+    this.updateFingerprintContract = (str) => {
+        this.fingerprintContract = this.hashString([this.fingerprintContract, str].join('-'));
+    };
+
+    this.setProfile = (profile, cb) => {
+        var lastProfile = this.profile;
+        if (profile !== lastProfile) {
+            this.profile = profile;
+            this.hashProfile((hash) => {
+                this.profileHash = hash;
+                this.updateFingerprintExtract(this.profileHash);
+                cb();
+            });
+        } else {
+            this.updateFingerprintExtract(this.profileHash);
+            cb();
+        }
+    };
+
+    this.setExtractArgs = (args, callback) => {
+        this.extractArgs = args;
+        this.updateFingerprintExtract(args);
+        callback();
+    };
+
+    this.setContractArgs = (args, callback) => {
+        this.contractArgs = args;
+        this.updateFingerprintContract(args);
+        callback();
+    };
+};
diff --git a/features/support/data.js b/features/support/data.js
index f0bf302d8bf..48e5d612c1b 100644
--- a/features/support/data.js
+++ b/features/support/data.js
@@ -1,14 +1,11 @@
-'use strict';
+var fs = require('fs');
+var path = require('path');
+var util = require('util');
+var exec = require('child_process').exec;
+var d3 = require('d3-queue');
 
-const fs = require('fs');
-const util = require('util');
-const d3 = require('d3-queue');
-
-const OSM = require('../lib/osm');
-const classes = require('./data_classes');
-const tableDiff = require('../lib/table_diff');
-const ensureDecimal = require('../lib/utils').ensureDecimal;
-const errorReason = require('../lib/utils').errorReason;
+var OSM = require('./build_osm');
+var classes = require('./data_classes');
 
 module.exports = function () {
     this.setGridSize = (meters) => {
@@ -97,8 +94,13 @@ module.exports = function () {
         q.awaitAll(callback);
     };
 
+    this.ensureDecimal = (i) => {
+        if (parseInt(i) === i) return i.toFixed(1);
+        else return i;
+    };
+
     this.tableCoordToLonLat = (ci, ri) => {
-        return [this.origin[0] + ci * this.zoom, this.origin[1] - ri * this.zoom].map(ensureDecimal);
+        return [this.origin[0] + ci * this.zoom, this.origin[1] - ri * this.zoom].map(this.ensureDecimal);
     };
 
     this.addOSMNode = (name, lon, lat, id) => {
@@ -130,6 +132,10 @@ module.exports = function () {
         return this.nameWayHash[s.toString()] || this.nameWayHash[s.toString().split('').reverse().join('')];
     };
 
+    this.resetData = () => {
+        this.resetOSM();
+    };
+
     this.makeOSMId = () => {
         this.osmID = this.osmID + 1;
         return this.osmID;
@@ -137,88 +143,206 @@ module.exports = function () {
 
     this.resetOSM = () => {
         this.OSMDB.clear();
+        this.osmData.reset();
         this.nameNodeHash = {};
         this.locationHash = {};
-        this.shortcutsHash = {};
         this.nameWayHash = {};
         this.osmID = 0;
     };
 
     this.writeOSM = (callback) => {
-        fs.exists(this.scenarioCacheFile, (exists) => {
-            if (exists) callback();
-            else {
-                this.OSMDB.toXML((xml) => {
-                    fs.writeFile(this.scenarioCacheFile, xml, callback);
+        fs.exists(this.DATA_FOLDER, (exists) => {
+            var mkDirFn = exists ? (cb) => { cb(); } : fs.mkdir.bind(fs.mkdir, this.DATA_FOLDER);
+            mkDirFn((err) => {
+                if (err) return callback(err);
+                var osmPath = path.resolve(this.DATA_FOLDER, util.format('%s.osm', this.osmData.osmFile));
+                fs.exists(osmPath, (exists) => {
+                    if (!exists) fs.writeFile(osmPath, this.osmData.str, callback);
+                    else callback();
                 });
-            }
+            });
         });
     };
 
-    this.linkOSM = (callback) => {
-        fs.exists(this.inputCacheFile, (exists) => {
-            if (exists) callback();
-            else {
-                fs.link(this.scenarioCacheFile, this.inputCacheFile, callback);
-            }
+    this.isExtracted = (callback) => {
+        fs.exists(util.format('%s.osrm', this.osmData.extractedFile), (core) => {
+            if (!core) return callback(false);
+            fs.exists(util.format('%s.osrm.names', this.osmData.extractedFile), (names) => {
+                if (!names) return callback(false);
+                fs.exists(util.format('%s.osrm.restrictions', this.osmData.extractedFile), (restrictions) => {
+                    return callback(restrictions);
+                });
+            });
         });
     };
 
-    this.extractData = (p, callback) => {
-        let stamp = p.processedCacheFile + '.extract';
-        fs.exists(stamp, (exists) => {
-            if (exists) return callback();
+    this.isContracted = (callback) => {
+        fs.exists(util.format('%s.osrm.hsgr', this.osmData.contractedFile), callback);
+    };
 
-            this.runBin('osrm-extract', util.format('%s --profile %s %s', p.extractArgs, p.profileFile, p.inputCacheFile), p.environment, (err) => {
-                if (err) {
-                    return callback(new Error(util.format('osrm-extract %s: %s', errorReason(err), err.cmd)));
-                }
-                fs.writeFile(stamp, 'ok', callback);
-            });
+    this.writeTimestamp = (callback) => {
+        fs.writeFile(util.format('%s.osrm.timestamp', this.osmData.contractedFile), this.OSM_TIMESTAMP, callback);
+    };
+
+    this.writeInputData = (callback) => {
+        this.writeOSM((err) => {
+            if (err) return callback(err);
+            this.writeTimestamp(callback);
         });
     };
 
-    this.contractData = (p, callback) => {
-        let stamp = p.processedCacheFile + '.contract';
-        fs.exists(stamp, (exists) => {
-            if (exists) return callback();
+    this.extractData = (callback) => {
+        this.logPreprocessInfo();
+        this.log(util.format('== Extracting %s.osm...', this.osmData.osmFile), 'preprocess');
+        var cmd = util.format('%s/osrm-extract %s.osm %s --profile %s/%s.lua >>%s 2>&1',
+            this.BIN_PATH, this.osmData.osmFile, this.extractArgs || '', this.PROFILES_PATH, this.profile, this.PREPROCESS_LOG_FILE);
+        this.log(cmd);
+        process.chdir(this.TEST_FOLDER);
+        exec(cmd, (err) => {
+            if (err) {
+                this.log(util.format('*** Exited with code %d', err.code), 'preprocess');
+                process.chdir('../');
+                return callback(this.ExtractError(err.code, util.format('osrm-extract exited with code %d', err.code)));
+            }
+
+            var q = d3.queue();
 
-            this.runBin('osrm-contract', util.format('%s %s', p.contractArgs, p.processedCacheFile), p.environment, (err) => {
-                if (err) {
-                    return callback(new Error(util.format('osrm-contract %s: %s', errorReason(err), err)));
-                }
-                fs.writeFile(stamp, 'ok', callback);
+            var rename = (file, cb) => {
+                this.log(util.format('Renaming %s.%s to %s.%s', this.osmData.osmFile, file, this.osmData.extractedFile, file), 'preprocess');
+                fs.rename([this.osmData.osmFile, file].join('.'), [this.osmData.extractedFile, file].join('.'), (err) => {
+                    if (err) return cb(this.FileError(null, 'failed to rename data file after extracting'));
+                    cb();
+                });
+            };
+
+            var renameIfExists = (file, cb) => {
+                fs.stat([this.osmData.osmFile, file].join('.'), (doesNotExistErr, exists) => {
+                    if (exists) rename(file, cb);
+                    else cb();
+                });
+            };
+
+            ['osrm', 'osrm.ebg', 'osrm.edges', 'osrm.enw', 'osrm.fileIndex', 'osrm.geometry', 'osrm.icd',
+             'osrm.names', 'osrm.nodes', 'osrm.properties', 'osrm.ramIndex', 'osrm.restrictions', 'osrm.tld', 'osrm.tls'].forEach(file => {
+                 q.defer(rename, file);
+             });
+
+            ['osrm.edge_penalties', 'osrm.edge_segment_lookup'].forEach(file => {
+                q.defer(renameIfExists, file);
+            });
+
+            q.awaitAll((err) => {
+                this.log('Finished extracting ' + this.osmData.extractedFile, 'preprocess');
+                process.chdir('../');
+                callback(err);
             });
         });
     };
 
-    this.extractAndContract = (callback) => {
-        // a shallow copy of scenario parameters to avoid data inconsistency
-        // if a cucumber timeout occurs during deferred jobs
-        let p = {extractArgs: this.extractArgs, contractArgs: this.contractArgs,
-                 profileFile: this.profileFile, inputCacheFile: this.inputCacheFile,
-                 processedCacheFile: this.processedCacheFile, environment: this.environment};
-        let queue = d3.queue(1);
-        queue.defer(this.extractData.bind(this), p);
-        queue.defer(this.contractData.bind(this), p);
-        queue.awaitAll(callback);
+    this.contractData = (callback) => {
+        this.logPreprocessInfo();
+        this.log(util.format('== Contracting %s.osm...', this.osmData.extractedFile), 'preprocess');
+        var cmd = util.format('%s/osrm-contract %s %s.osrm >>%s 2>&1',
+            this.BIN_PATH, this.contractArgs || '', this.osmData.extractedFile, this.PREPROCESS_LOG_FILE);
+        this.log(cmd);
+        process.chdir(this.TEST_FOLDER);
+        exec(cmd, (err) => {
+            if (err) {
+                this.log(util.format('*** Exited with code %d', err.code), 'preprocess');
+                process.chdir('../');
+                return callback(this.ContractError(err.code, util.format('osrm-contract exited with code %d', err.code)));
+            }
+
+            var rename = (file, cb) => {
+                this.log(util.format('Renaming %s.%s to %s.%s', this.osmData.extractedFile, file, this.osmData.contractedFile, file), 'preprocess');
+                fs.rename([this.osmData.extractedFile, file].join('.'), [this.osmData.contractedFile, file].join('.'), (err) => {
+                    if (err) return cb(this.FileError(null, 'failed to rename data file after contracting.'));
+                    cb();
+                });
+            };
+
+            var renameIfExists = (file, cb) => {
+                fs.stat([this.osmData.extractedFile, file].join('.'), (doesNotExistErr, exists) => {
+                    if (exists) rename(file, cb);
+                    else cb();
+                });
+            };
+
+            var copy = (file, cb) => {
+                this.log(util.format('Copying %s.%s to %s.%s', this.osmData.extractedFile, file, this.osmData.contractedFile, file), 'preprocess');
+                fs.createReadStream([this.osmData.extractedFile, file].join('.'))
+                    .pipe(fs.createWriteStream([this.osmData.contractedFile, file].join('.'))
+                            .on('finish', cb)
+                        )
+                    .on('error', () => {
+                        return cb(this.FileError(null, 'failed to copy data after contracting.'));
+                    });
+            };
+
+            var q = d3.queue();
+
+            ['osrm', 'osrm.core', 'osrm.datasource_indexes', 'osrm.datasource_names', 'osrm.ebg','osrm.edges',
+             'osrm.enw', 'osrm.fileIndex', 'osrm.geometry', 'osrm.hsgr', 'osrm.icd','osrm.level', 'osrm.names',
+             'osrm.nodes', 'osrm.properties', 'osrm.ramIndex', 'osrm.restrictions', 'osrm.tld', 'osrm.tls'].forEach((file) => {
+                 q.defer(rename, file);
+             });
+
+            ['osrm.edge_penalties', 'osrm.edge_segment_lookup'].forEach(file => {
+                q.defer(renameIfExists, file);
+            });
+
+            [].forEach((file) => {
+                q.defer(copy, file);
+            });
+
+            q.awaitAll((err) => {
+                this.log('Finished contracting ' + this.osmData.contractedFile, 'preprocess');
+                process.chdir('../');
+                callback(err);
+            });
+        });
     };
 
+    var noop = (cb) => cb();
+
     this.reprocess = (callback) => {
-        let queue = d3.queue(1);
-        queue.defer(this.writeOSM.bind(this));
-        queue.defer(this.linkOSM.bind(this));
-        queue.defer(this.extractAndContract.bind(this));
-        queue.awaitAll(callback);
+        this.osmData.populate(() => {
+            this.isContracted((isContracted) => {
+                if (!isContracted) {
+                    this.writeAndExtract((e) => {
+                        if (e) return callback(e);
+                        this.contractData((e) => {
+                            if (e) return callback(e);
+                            this.logPreprocessDone();
+                            callback();
+                        });
+                    });
+                } else {
+                    this.log('Already contracted ' + this.osmData.contractedFile, 'preprocess');
+                    callback();
+                }
+            });
+        });
+    };
+
+    this.writeAndExtract = (callback) => {
+        this.writeInputData((e) => {
+            if (e) return callback(e);
+            this.isExtracted((isExtracted) => {
+                var extractFn = isExtracted ? noop : this.extractData;
+                if (isExtracted) this.log('Already extracted ' + this.osmData.extractedFile, 'preprocess');
+                extractFn((e) => {
+                    callback(e);
+                });
+            });
+        });
     };
 
     this.reprocessAndLoadData = (callback) => {
-        let queue = d3.queue(1);
-        queue.defer(this.writeOSM.bind(this));
-        queue.defer(this.linkOSM.bind(this));
-        queue.defer(this.extractAndContract.bind(this));
-        queue.defer(this.osrmLoader.load.bind(this.osrmLoader), this.processedCacheFile);
-        queue.awaitAll(callback);
+        this.reprocess((e) => {
+            if (e) return callback(e);
+            this.OSRMLoader.load(util.format('%s.osrm', this.osmData.contractedFile), callback);
+        });
     };
 
     this.processRowsAndDiff = (table, fn, callback) => {
@@ -228,9 +352,7 @@ module.exports = function () {
 
         q.awaitAll((err, actual) => {
             if (err) return callback(err);
-            let diff = tableDiff(table, actual);
-            if (diff) callback(new Error(diff));
-            else callback();
+            this.diffTables(table, actual, {}, callback);
         });
     };
 };
diff --git a/features/support/data_classes.js b/features/support/data_classes.js
index a17141e5e7d..391bb1c6de3 100644
--- a/features/support/data_classes.js
+++ b/features/support/data_classes.js
@@ -1,6 +1,7 @@
 'use strict';
 
-const util = require('util');
+var util = require('util');
+var path = require('path');
 
 module.exports = {
     Location: class {
@@ -10,6 +11,43 @@ module.exports = {
         }
     },
 
+    osmData: class {
+        constructor (scope) {
+            this.scope          = scope;
+            this.str            = null;
+            this.hash           = null;
+            this.fingerprintOSM = null;
+            this.osmFile        = null;
+            this.extractedFile  = null;
+            this.contractedFile   = null;
+        }
+
+        populate (callback) {
+            this.scope.OSMDB.toXML((str) => {
+                this.str = str;
+
+                this.hash = this.scope.hashString(str);
+                this.fingerprintOSM = this.scope.hashString(this.hash);
+
+                this.osmFile        = path.resolve(this.scope.DATA_FOLDER, this.fingerprintOSM);
+
+                this.extractedFile  = path.resolve([this.osmFile, this.scope.fingerprintExtract].join('_'));
+                this.contractedFile   = path.resolve([this.osmFile, this.scope.fingerprintExtract, this.scope.fingerprintContract].join('_'));
+
+                callback();
+            });
+        }
+
+        reset () {
+            this.str            = null;
+            this.hash           = null;
+            this.fingerprintOSM = null;
+            this.osmFile        = null;
+            this.extractedFile  = null;
+            this.contractedFile   = null;
+        }
+    },
+
     FuzzyMatch: class {
         match (got, want) {
             var matchPercent = want.match(/(.*)\s+~(.+)%$/),
diff --git a/features/support/env.js b/features/support/env.js
index 564579ee939..c99d5440312 100644
--- a/features/support/env.js
+++ b/features/support/env.js
@@ -1,46 +1,32 @@
-'use strict';
+var path = require('path');
+var util = require('util');
+var fs = require('fs');
+var exec = require('child_process').exec;
+var d3 = require('d3-queue');
 
-const path = require('path');
-const util = require('util');
-const fs = require('fs');
-const d3 = require('d3-queue');
-const child_process = require('child_process');
-const tryConnect = require('../lib/try_connect');
-
-// Sets up all constants that are valid for all features
 module.exports = function () {
     this.initializeEnv = (callback) => {
+        this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000;
         this.TIMEOUT = process.env.CUCUMBER_TIMEOUT && parseInt(process.env.CUCUMBER_TIMEOUT) || 5000;
-        // set cucumber default timeout
         this.setDefaultTimeout(this.TIMEOUT);
-        this.ROOT_PATH = process.cwd();
-
-        this.TEST_PATH = path.resolve(this.ROOT_PATH, 'test');
-        this.CACHE_PATH = path.resolve(this.TEST_PATH, 'cache');
-        this.LOGS_PATH = path.resolve(this.TEST_PATH, 'logs');
-
-        this.PROFILES_PATH = path.resolve(this.ROOT_PATH, 'profiles');
-        this.FIXTURES_PATH = path.resolve(this.ROOT_PATH, 'unit_tests/fixtures');
-        this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_PATH, 'build');
-        var stxxl_config = path.resolve(this.ROOT_PATH, 'test/.stxxl');
-        if (!fs.existsSync(stxxl_config)) {
-            return callback(new Error('*** '+stxxl_config+ 'does not exist'));
-        }
-
-        this.DEFAULT_ENVIRONMENT = Object.assign({STXXLCFG: stxxl_config}, process.env);
-        this.DEFAULT_PROFILE = 'bicycle';
-        this.DEFAULT_INPUT_FORMAT = 'osm';
-        this.DEFAULT_LOAD_METHOD = 'datastore';
-        this.DEFAULT_ORIGIN = [1,1];
+        this.ROOT_FOLDER = process.cwd();
         this.OSM_USER = 'osrm';
         this.OSM_GENERATOR = 'osrm-test';
         this.OSM_UID = 1;
+        this.TEST_FOLDER = path.resolve(this.ROOT_FOLDER, 'test');
+        this.DATA_FOLDER = path.resolve(this.TEST_FOLDER, 'cache');
         this.OSM_TIMESTAMP = '2000-01-01T00:00:00Z';
+        this.DEFAULT_SPEEDPROFILE = 'bicycle';
         this.WAY_SPACING = 100;
-        this.DEFAULT_GRID_SIZE = 100; // meters
-
-        this.OSRM_PORT = process.env.OSRM_PORT && parseInt(process.env.OSRM_PORT) || 5000;
-        this.HOST = 'http://127.0.0.1:' + this.OSRM_PORT;
+        this.DEFAULT_GRID_SIZE = 100;    // meters
+        this.PROFILES_PATH = path.resolve(this.ROOT_FOLDER, 'profiles');
+        this.FIXTURES_PATH = path.resolve(this.ROOT_FOLDER, 'unit_tests/fixtures');
+        this.BIN_PATH = process.env.OSRM_BUILD_DIR && process.env.OSRM_BUILD_DIR || path.resolve(this.ROOT_FOLDER, 'build');
+        this.DEFAULT_INPUT_FORMAT = 'osm';
+        this.DEFAULT_ORIGIN = [1,1];
+        this.DEFAULT_LOAD_METHOD = 'datastore';
+        this.OSRM_ROUTED_LOG_FILE = path.resolve(this.TEST_FOLDER, 'osrm-routed.log');
+        this.ERROR_LOG_FILE = path.resolve(this.TEST_FOLDER, 'error.log');
 
         // TODO make sure this works on win
         if (process.platform.match(/indows.*/)) {
@@ -51,49 +37,36 @@ module.exports = function () {
         } else {
             this.TERMSIGNAL = 'SIGTERM';
             this.EXE = '';
-            // TODO autodetect if this was build with shared or static libraries
-            this.LIB = process.env.BUILD_SHARED_LIBS && '.so' || '.a';
+            this.LIB = '.so';
             this.QQ = '';
         }
 
-        this.OSRM_EXTRACT_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-extract', this.EXE));
-        this.OSRM_CONTRACT_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-contract', this.EXE));
-        this.OSRM_ROUTED_PATH = path.resolve(util.format('%s/%s%s', this.BIN_PATH, 'osrm-routed', this.EXE));
-        this.LIB_OSRM_EXTRACT_PATH = util.format('%s/libosrm_extract%s', this.BIN_PATH, this.LIB),
-        this.LIB_OSRM_CONTRACT_PATH = util.format('%s/libosrm_contract%s', this.BIN_PATH, this.LIB),
-        this.LIB_OSRM_PATH = util.format('%s/libosrm%s', this.BIN_PATH, this.LIB);
-
         // eslint-disable-next-line no-console
         console.info(util.format('Node Version', process.version));
         if (parseInt(process.version.match(/v(\d)/)[1]) < 4) throw new Error('*** PLease upgrade to Node 4.+ to run OSRM cucumber tests');
 
-        fs.exists(this.TEST_PATH, (exists) => {
-            if (exists)
-                return callback();
-            else
-                return callback(new Error('*** Test folder doesn\'t exist.'));
+        fs.exists(this.TEST_FOLDER, (exists) => {
+            if (!exists) throw new Error(util.format('*** Test folder %s doesn\'t exist.', this.TEST_FOLDER));
+            callback();
         });
     };
 
-    this.getProfilePath = (profile) => {
-        return path.resolve(this.PROFILES_PATH, profile + '.lua');
-    };
-
-    this.verifyOSRMIsNotRunning = (callback) => {
-        tryConnect(this.OSRM_PORT, (err) => {
-            if (!err) return callback(new Error('*** osrm-routed is already running.'));
-            else callback();
-        });
+    this.verifyOSRMIsNotRunning = () => {
+        if (this.OSRMLoader.up()) {
+            throw new Error('*** osrm-routed is already running.');
+        }
     };
 
     this.verifyExistenceOfBinaries = (callback) => {
-        var verify = (binPath, cb) => {
+        var verify = (bin, cb) => {
+            var binPath = path.resolve(util.format('%s/%s%s', this.BIN_PATH, bin, this.EXE));
             fs.exists(binPath, (exists) => {
-                if (!exists) return cb(new Error(util.format('%s is missing. Build failed?', binPath)));
+                if (!exists) throw new Error(util.format('%s is missing. Build failed?', binPath));
                 var helpPath = util.format('%s --help > /dev/null 2>&1', binPath);
-                child_process.exec(helpPath, (err) => {
+                exec(helpPath, (err) => {
                     if (err) {
-                        return cb(new Error(util.format('*** %s exited with code %d', helpPath, err.code)));
+                        this.log(util.format('*** Exited with code %d', err.code), 'preprocess');
+                        throw new Error(util.format('*** %s exited with code %d', helpPath, err.code));
                     }
                     cb();
                 });
@@ -101,12 +74,23 @@ module.exports = function () {
         };
 
         var q = d3.queue();
-        [this.OSRM_EXTRACT_PATH, this.OSRM_CONTRACT_PATH, this.OSRM_ROUTED_PATH].forEach(bin => { q.defer(verify, bin); });
-        q.awaitAll(callback);
+        ['osrm-extract', 'osrm-contract', 'osrm-routed'].forEach(bin => { q.defer(verify, bin); });
+        q.awaitAll(() => {
+            callback();
+        });
+    };
+
+    this.AfterConfiguration = (callback) => {
+        this.clearLogFiles(() => {
+            this.verifyOSRMIsNotRunning();
+            this.verifyExistenceOfBinaries(() => {
+                callback();
+            });
+        });
     };
 
     process.on('exit', () => {
-        this.osrmLoader.shutdown(() => {});
+        if (this.OSRMLoader.loader) this.OSRMLoader.shutdown(() => {});
     });
 
     process.on('SIGINT', () => {
diff --git a/features/support/exception_classes.js b/features/support/exception_classes.js
new file mode 100644
index 00000000000..36bdffe8b53
--- /dev/null
+++ b/features/support/exception_classes.js
@@ -0,0 +1,132 @@
+'use strict';
+
+var util = require('util');
+var path = require('path');
+var fs = require('fs');
+var chalk = require('chalk');
+
+var OSRMError = class extends Error {
+    constructor (process, code, msg, log, lines) {
+        super(msg);
+        this.process = process;
+        this.code = code;
+        this.msg = msg;
+        this.lines = lines;
+        this.log = log;
+    }
+
+    extract (callback) {
+        this.logTail(this.log, this.lines, callback);
+    }
+
+    // toString (callback) {
+    //     this.extract((tail) => {
+    //         callback(util.format('*** %s\nLast %s from %s:\n%s\n', this.msg, this.lines, this.log, tail));
+    //     });
+    // }
+
+    logTail (logPath, n, callback) {
+        var expanded = path.resolve(this.TEST_FOLDER, logPath);
+        fs.exists(expanded, (exists) => {
+            if (exists) {
+                fs.readFile(expanded, (err, data) => {
+                    var lines = data.toString().trim().split('\n');
+                    callback(lines
+                        .slice(lines.length - n)
+                        .map(line => util.format('    %s', line))
+                        .join('\n'));
+                });
+            } else {
+                callback(util.format('File %s does not exist!', expanded));
+            }
+        });
+    }
+};
+
+var unescapeStr = (str) => str.replace(/\\\|/g, '\|').replace(/\\\\/g, '\\');
+
+module.exports = {
+    OSRMError: OSRMError,
+
+    FileError: class extends OSRMError {
+        constructor (logFile, code, msg) {
+            super ('fileutil', code, msg, logFile, 5);
+        }
+    },
+
+    LaunchError: class extends OSRMError {
+        constructor (logFile, launchProcess, code, msg) {
+            super (launchProcess, code, msg, logFile, 5);
+        }
+    },
+
+    ExtractError: class extends OSRMError {
+        constructor (logFile, code, msg) {
+            super('osrm-extract', code, msg, logFile, 3);
+        }
+    },
+
+    ContractError:  class extends OSRMError {
+        constructor (logFile, code, msg) {
+            super('osrm-contract', code, msg, logFile, 3);
+        }
+    },
+
+    RoutedError: class extends OSRMError {
+        constructor (logFile, msg) {
+            super('osrm-routed', null, msg, logFile, 3);
+        }
+    },
+
+    TableDiffError: class extends Error {
+        constructor (expected, actual) {
+            super();
+            this.headers = expected.raw()[0];
+            this.expected = expected.hashes();
+            this.actual = actual;
+            this.diff = [];
+            this.hasErrors = false;
+
+            var good = 0, bad = 0;
+
+            this.expected.forEach((row, i) => {
+                var rowError = false;
+
+                for (var j in row) {
+                    if (unescapeStr(row[j]) != actual[i][j]) {
+                        rowError = true;
+                        this.hasErrors = true;
+                        break;
+                    }
+                }
+
+                if (rowError) {
+                    bad++;
+                    this.diff.push(Object.assign({}, row, {c_status: 'undefined'}));
+                    this.diff.push(Object.assign({}, actual[i], {c_status: 'comment'}));
+                } else {
+                    good++;
+                    this.diff.push(row);
+                }
+            });
+        }
+
+        get string () {
+            if (!this.hasErrors) return null;
+
+            var s = ['Tables were not identical:'];
+            s.push(this.headers.map(key => '    ' + key).join(' | '));
+            this.diff.forEach((row) => {
+                var rowString = '| ';
+                this.headers.forEach((header) => {
+                    if (!row.c_status) rowString += chalk.green('    ' + row[header] + ' | ');
+                    else if (row.c_status === 'undefined') rowString += chalk.yellow('(-) ' + row[header] + ' | ');
+                    else rowString += chalk.red('(+) ' + row[header] + ' | ');
+                });
+                s.push(rowString);
+            });
+
+            return s.join('\n') + '\nTODO this is a temp workaround waiting for https://github.com/cucumber/cucumber-js/issues/534';
+        }
+    }
+};
diff --git a/features/support/exceptions.js b/features/support/exceptions.js
new file mode 100644
index 00000000000..6af1a93b0ad
--- /dev/null
+++ b/features/support/exceptions.js
@@ -0,0 +1,15 @@
+var exceptions = require('./exception_classes');
+
+module.exports = function () {
+    this.OSRMError = exceptions.OSRMError,
+
+    this.FileError = (code, msg) => new (exceptions.FileError.bind(exceptions.FileError, this.PREPROCESS_LOG_FILE))(code, msg);
+
+    this.LaunchError = (code, launchProcess, msg) => new (exceptions.LaunchError.bind(exceptions.LaunchError, this.ERROR_LOG_FILE))(code, launchProcess, msg);
+
+    this.ExtractError = (code, msg) => new (exceptions.ExtractError.bind(exceptions.ExtractError, this.PREPROCESS_LOG_FILE))(code, msg);
+
+    this.ContractError = (code, msg) => new (exceptions.ContractError.bind(exceptions.ContractError, this.PREPROCESS_LOG_FILE))(code, msg);
+
+    this.RoutedError = (msg) => new (exceptions.RoutedError.bind(exceptions.RoutedError, this.OSRM_ROUTED_LOG_FILE))(msg);
+};
diff --git a/features/support/hash.js b/features/support/hash.js
new file mode 100644
index 00000000000..399dd51cac5
--- /dev/null
+++ b/features/support/hash.js
@@ -0,0 +1,43 @@
+var fs = require('fs');
+var path = require('path');
+var crypto = require('crypto');
+var d3 = require('d3-queue');
+
+module.exports = function () {
+    this.hashOfFiles = (paths, cb) => {
+        paths = Array.isArray(paths) ? paths : [paths];
+        var shasum = crypto.createHash('sha1'), hashedFiles = false;
+
+        var q = d3.queue(1);
+
+        var addFile = (path, cb) => {
+            fs.readFile(path, (err, data) => {
+                if (err && err.code === 'ENOENT') cb(); // ignore non-existing files
+                else if (err) cb(err);
+                else {
+                    shasum.update(data);
+                    hashedFiles = true;
+                    cb();
+                }
+            });
+        };
+
+        paths.forEach(path => { q.defer(addFile, path); });
+
+        q.awaitAll(err => {
+            if (err) throw new Error('*** Error reading files:', err);
+            if (!hashedFiles) throw new Error('*** No files found: [' + paths.join(', ') + ']');
+            cb(shasum.digest('hex'));
+        });
+    };
+
+    this.hashProfile = (cb) => {
+        this.hashOfFiles(path.resolve(this.PROFILES_PATH, this.profile + '.lua'), cb);
+    };
+
+    this.hashString = (str) => {
+        return crypto.createHash('sha1').update(str).digest('hex');
+    };
+
+    return this;
+};
diff --git a/features/support/hooks.js b/features/support/hooks.js
index b0f6b0b77f8..1e265ea14ec 100644
--- a/features/support/hooks.js
+++ b/features/support/hooks.js
@@ -1,61 +1,36 @@
-'use strict';
-
-var d3 = require('d3-queue');
-var path = require('path');
-var mkdirp = require('mkdirp');
-var rimraf = require('rimraf');
-var OSM = require('../lib/osm');
-var OSRMLoader = require('../lib/osrm_loader');
+var util = require('util');
 
 module.exports = function () {
-    this.registerHandler('BeforeFeatures', {timeout: 30000},  (features, callback) => {
-        this.osrmLoader = new OSRMLoader(this);
-        this.OSMDB = new OSM.DB();
-
-        let queue = d3.queue(1);
-        queue.defer(this.initializeEnv.bind(this));
-        queue.defer(this.verifyOSRMIsNotRunning.bind(this));
-        queue.defer(this.verifyExistenceOfBinaries.bind(this));
-        queue.defer(this.initializeCache.bind(this));
-        queue.defer(this.setupFeatures.bind(this, features));
-        queue.awaitAll(callback);
-    });
-
-    this.BeforeFeature((feature, callback) => {
-        this.profile = this.DEFAULT_PROFILE;
-        this.profileFile = path.join(this.PROFILES_PATH, this.profile + '.lua');
-        this.setupFeatureCache(feature);
-        callback();
+    this.BeforeFeatures((features, callback) => {
+        this.pid = null;
+        this.initializeEnv(() => {
+            this.initializeOptions(callback);
+        });
     });
 
     this.Before((scenario, callback) => {
-        this.osrmLoader.setLoadMethod(this.DEFAULT_LOAD_METHOD);
+        this.scenarioTitle = scenario.getName();
+
+        this.loadMethod = this.DEFAULT_LOAD_METHOD;
+        this.queryParams = {};
+        var d = new Date();
+        this.scenarioTime = util.format('%d-%d-%dT%s:%s:%sZ', d.getFullYear(), d.getMonth()+1, d.getDate(), d.getHours(), d.getMinutes(), d.getSeconds());
+        this.resetData();
+        this.hasLoggedPreprocessInfo = false;
+        this.hasLoggedScenarioInfo = false;
         this.setGridSize(this.DEFAULT_GRID_SIZE);
         this.setOrigin(this.DEFAULT_ORIGIN);
-        this.queryParams = {};
-        this.extractArgs = '';
-        this.contractArgs = '';
-        this.environment = Object.assign(this.DEFAULT_ENVIRONMENT);
-        this.resetOSM();
-
-        this.scenarioID = this.getScenarioID(scenario);
-        this.setupScenarioCache(this.scenarioID);
-
-        // setup output logging
-        let logDir = path.join(this.LOGS_PATH, this.featureID);
-        this.scenarioLogFile = path.join(logDir, this.scenarioID) + '.log';
-        d3.queue(1)
-            .defer(mkdirp, logDir)
-            .defer(rimraf, this.scenarioLogFile)
-            .awaitAll(callback);
-    });
-
-    this.After((scenario, callback) => {
-        this.resetOptionsOutput();
+        this.fingerprintExtract = this.hashString([this.luaLibHash, this.binExtractHash].join('-'));
+        this.fingerprintContract = this.hashString(this.binContractHash);
         callback();
     });
 
-    this.AfterFeatures((features, callback) => {
-        callback();
+    this.After((scenario, callback) => {
+        this.setExtractArgs('', () => {
+            this.setContractArgs('', () => {
+                if (this.loadMethod === 'directly' && !!this.OSRMLoader.loader) this.OSRMLoader.shutdown(callback);
+                else callback();
+            });
+        });
     });
 };
diff --git a/features/support/http.js b/features/support/http.js
index 71f61761a45..3ae2edc9301 100644
--- a/features/support/http.js
+++ b/features/support/http.js
@@ -19,9 +19,6 @@ module.exports = function () {
         return paramString;
     };
 
-    // FIXME this needs to be simplified!
-    // - remove usage of node-timeout
-    // - replace with node's native timout mechanism
     this.sendRequest = (baseUri, parameters, callback) => {
         var limit = Timeout(this.TIMEOUT, { err: { statusCode: 408 } });
 
@@ -31,9 +28,9 @@ module.exports = function () {
 
             request(this.query, (err, res, body) => {
                 if (err && err.code === 'ECONNREFUSED') {
-                    return cb(new Error('*** osrm-routed is not running.'));
+                    throw new Error('*** osrm-routed is not running.');
                 } else if (err && err.statusCode === 408) {
-                    return cb(new Error());
+                    throw new Error();
                 }
 
                 return cb(err, res, body);
@@ -43,10 +40,11 @@ module.exports = function () {
         runRequest(limit((err, res, body) => {
             if (err) {
                 if (err.statusCode === 408)
-                    return callback(new Error('*** osrm-routed did not respond'));
+                    return callback(this.RoutedError('*** osrm-routed did not respond'));
                 else if (err.code === 'ECONNREFUSED')
-                    return callback(new Error('*** osrm-routed is not running'));
+                    return callback(this.RoutedError('*** osrm-routed is not running'));
             }
+            //console.log(body+"\n");
             return callback(err, res, body);
         }));
     };
diff --git a/features/support/launch.js b/features/support/launch.js
new file mode 100644
index 00000000000..ee335e301f5
--- /dev/null
+++ b/features/support/launch.js
@@ -0,0 +1,5 @@
+var launchClasses = require('./launch_classes');
+
+module.exports = function () {
+    this._OSRMLoader = () => new (launchClasses._OSRMLoader.bind(launchClasses._OSRMLoader, this))();
+};
diff --git a/features/support/launch_classes.js b/features/support/launch_classes.js
new file mode 100644
index 00000000000..2aace884613
--- /dev/null
+++ b/features/support/launch_classes.js
@@ -0,0 +1,164 @@
+'use strict';
+
+var fs = require('fs');
+var spawn = require('child_process').spawn;
+var util = require('util');
+var net = require('net');
+var Timeout = require('node-timeout');
+
+var OSRMBaseLoader = class {
+    constructor (scope) {
+        this.scope = scope;
+    }
+
+    launch (callback) {
+        var limit = Timeout(this.scope.TIMEOUT, { err: this.scope.RoutedError('Launching osrm-routed timed out.') });
+
+        var runLaunch = (cb) => {
+            this.osrmUp(() => { this.waitForConnection(cb); });
+        };
+
+        runLaunch(limit((e) => { if (e) callback(e); else callback(); }));
+    }
+
+    shutdown (callback) {
+        var limit = Timeout(this.scope.TIMEOUT, { err: this.scope.RoutedError('Shutting down osrm-routed timed out.')});
+
+        var runShutdown = (cb) => {
+            this.osrmDown(cb);
+        };
+
+        runShutdown(limit((e) => { if (e) callback(e); else callback(); }));
+    }
+
+    osrmIsRunning () {
+        return !!this.scope.pid && this.child && !this.child.killed;
+    }
+
+    osrmDown (callback) {
+        if (this.scope.pid) {
+            process.kill(this.scope.pid, this.scope.TERMSIGNAL);
+            this.waitForShutdown(callback);
+            this.scope.pid = null;
+        } else callback(true);
+    }
+
+    waitForConnection (callback) {
+        var retryCount = 0;
+        var connectWithRetry = () => {
+            net.connect({ port: this.scope.OSRM_PORT, host: '127.0.0.1' })
+               .on('connect', () => { callback(); })
+               .on('error', () => {
+                   if (retryCount < 2) {
+                       retryCount++;
+                       setTimeout(connectWithRetry, 100);
+                   } else {
+                       callback(new Error('Could not connect to osrm-routed after three retires'));
+                   }
+               });
+        };
+
+        connectWithRetry();
+    }
+
+    waitForShutdown (callback) {
+        var check = () => {
+            if (!this.osrmIsRunning()) return callback();
+        };
+        setTimeout(check, 100);
+    }
+};
+
+var OSRMDirectLoader = class extends OSRMBaseLoader {
+    constructor (scope) {
+        super(scope);
+    }
+
+    load (inputFile, callback) {
+        this.inputFile = inputFile;
+        this.shutdown(() => {
+            this.launch(callback);
+        });
+    }
+
+    osrmUp (callback) {
+        if (this.scope.pid) return callback();
+        var writeToLog = (data) => {
+            fs.appendFile(this.scope.OSRM_ROUTED_LOG_FILE, data, (err) => { if (err) throw err; });
+        };
+
+        var child = spawn(util.format('%s/osrm-routed', this.scope.BIN_PATH), [this.inputFile, util.format('-p%d', this.scope.OSRM_PORT)]);
+        this.scope.pid = child.pid;
+        child.stdout.on('data', writeToLog);
+        child.stderr.on('data', writeToLog);
+
+        callback();
+    }
+};
+
+var OSRMDatastoreLoader = class extends OSRMBaseLoader {
+    constructor (scope) {
+        super(scope);
+    }
+
+    load (inputFile, callback) {
+        this.inputFile = inputFile;
+        this.loadData((err) => {
+            if (err) return callback(err);
+            if (!this.scope.pid) return this.launch(callback);
+            else callback();
+        });
+    }
+
+    loadData (callback) {
+        this.scope.runBin('osrm-datastore', this.inputFile, (err) => {
+            if (err) return callback(this.scope.LaunchError(this.exitCode, 'datastore', err));
+            callback();
+        });
+    }
+
+    osrmUp (callback) {
+        if (this.scope.pid) return callback();
+        var writeToLog = (data) => {
+            fs.appendFile(this.scope.OSRM_ROUTED_LOG_FILE, data, (err) => { if (err) throw err; });
+        };
+
+        var child = spawn(util.format('%s/osrm-routed', this.scope.BIN_PATH), ['--shared-memory=1', util.format('-p%d', this.scope.OSRM_PORT)]);
+        this.child = child;
+        this.scope.pid = child.pid;
+        child.stdout.on('data', writeToLog);
+        child.stderr.on('data', writeToLog);
+
+        callback();
+    }
+};
+
+module.exports = {
+    _OSRMLoader: class {
+        constructor (scope) {
+            this.scope = scope;
+            this.loader = null;
+        }
+
+        load (inputFile, callback) {
+            var method = this.scope.loadMethod;
+            if (method === 'datastore') {
+                this.loader = new OSRMDatastoreLoader(this.scope);
+                this.loader.load(inputFile, callback);
+            } else if (method === 'directly') {
+                this.loader = new OSRMDirectLoader(this.scope);
+                this.loader.load(inputFile, callback);
+            } else {
+                callback(new Error('*** Unknown load method ' + method));
+            }
+        }
+
+        shutdown (callback) {
+            this.loader.shutdown(callback);
+        }
+
+        up () {
+            return this.loader ? this.loader.osrmIsRunning() : false;
+        }
+    }
+};
diff --git a/features/support/log.js b/features/support/log.js
new file mode 100644
index 00000000000..c428cb9d985
--- /dev/null
+++ b/features/support/log.js
@@ -0,0 +1,90 @@
+var fs = require('fs');
+
+module.exports = function () {
+    this.clearLogFiles = (callback) => {
+        // emptying existing files, rather than deleting and writing new ones makes it
+        // easier to use tail -f from the command line
+        fs.writeFile(this.OSRM_ROUTED_LOG_FILE, '', err => {
+            if (err) throw err;
+            fs.writeFile(this.PREPROCESS_LOG_FILE, '', err => {
+                if (err) throw err;
+                fs.writeFile(this.LOG_FILE, '', err => {
+                    if (err) throw err;
+                    callback();
+                });
+            });
+        });
+    };
+
+    var log = this.log = (s, type) => {
+        s = s || '';
+        type = type || null;
+        var file = type === 'preprocess' ? this.PREPROCESS_LOG_FILE : this.LOG_FILE;
+        fs.appendFile(file, s + '\n', err => {
+            if (err) throw err;
+        });
+    };
+
+    this.logScenarioFailInfo = () => {
+        if (this.hasLoggedScenarioInfo) return;
+
+        log('=========================================');
+        log('Failed scenario: ' + this.scenarioTitle);
+        log('Time: ' + this.scenarioTime);
+        log('Fingerprint osm stage: ' + this.osmData.fingerprintOSM);
+        log('Fingerprint extract stage: ' + this.fingerprintExtract);
+        log('Fingerprint contract stage: ' + this.fingerprintContract);
+        log('Fingerprint route stage: ' + this.fingerprintRoute);
+        log('Profile: ' + this.profile);
+        log();
+        log('```xml');               // so output can be posted directly to github comment fields
+        log(this.osmData.str.trim());
+        log('```');
+        log();
+        log();
+
+        this.hasLoggedScenarioInfo = true;
+    };
+
+    this.logFail = (expected, got, attempts) => {
+        this.logScenarioFailInfo();
+        log('== ');
+        log('Expected: ' + JSON.stringify(expected));
+        log('Got:      ' + JSON.stringify(got));
+        log();
+        ['route','forw','backw'].forEach((direction) => {
+            if (attempts[direction]) {
+                log('Direction: ' + direction);
+                log('Query: ' + attempts[direction].query);
+                log('Response: ' + attempts[direction].response.body);
+                log();
+            }
+        });
+    };
+
+    this.logPreprocessInfo = () => {
+        if (this.hasLoggedPreprocessInfo) return;
+        log('=========================================', 'preprocess');
+        log('Preprocessing data for scenario: ' + this.scenarioTitle, 'preprocess');
+        log('Time: ' + this.scenarioTime, 'preprocess');
+        log('', 'preprocess');
+        log('== OSM data:', 'preprocess');
+        log('```xml', 'preprocess');            // so output can be posted directly to github comment fields
+        log(this.osmData.str, 'preprocess');
+        log('```', 'preprocess');
+        log('', 'preprocess');
+        log('== Profile:', 'preprocess');
+        log(this.profile, 'preprocess');
+        log('', 'preprocess');
+        this.hasLoggedPreprocessInfo = true;
+    };
+
+    this.logPreprocess = (str) => {
+        this.logPreprocessInfo();
+        log(str, 'preprocess');
+    };
+
+    this.logPreprocessDone = () => {
+        log('Done with preprocessing at ' + new Date(), 'preprocess');
+    };
+};
diff --git a/features/support/route.js b/features/support/route.js
index 76ce70e433e..408a2a4e5d8 100644
--- a/features/support/route.js
+++ b/features/support/route.js
@@ -1,8 +1,7 @@
 'use strict';
 
-const Timeout = require('node-timeout');
-const request = require('request');
-const ensureDecimal = require('../lib/utils').ensureDecimal;
+var Timeout = require('node-timeout');
+var request = require('request');
 
 module.exports = function () {
     this.requestPath = (service, params, callback) => {
@@ -43,7 +42,7 @@ module.exports = function () {
     };
 
     var encodeWaypoints = (waypoints) => {
-        return waypoints.map(w => [w.lon, w.lat].map(ensureDecimal).join(','));
+        return waypoints.map(w => [w.lon, w.lat].map(this.ensureDecimal).join(','));
     };
 
     this.requestRoute = (waypoints, bearings, userParams, callback) => {
diff --git a/features/support/run.js b/features/support/run.js
index 8c4d7fdc296..35561d80301 100644
--- a/features/support/run.js
+++ b/features/support/run.js
@@ -1,52 +1,40 @@
-'use strict';
-
-const fs = require('fs');
-const util = require('util');
-const child_process = require('child_process');
+var fs = require('fs');
+var util = require('util');
+var exec = require('child_process').exec;
 
 module.exports = function () {
-    // replaces placeholders for in user supplied commands
-    this.expandOptions = (options) => {
-        let opts = options.slice();
-        let table = {
-            '{osm_file}': this.inputCacheFile,
-            '{processed_file}': this.processedCacheFile,
-            '{profile_file}': this.profileFile,
-            '{rastersource_file}': this.rasterCacheFile,
-            '{speeds_file}': this.speedsCacheFile,
-            '{penalties_file}': this.penaltiesCacheFile
-        };
+    this.runBin = (bin, options, callback) => {
+        var opts = options.slice();
 
-        for (let k in table) {
-            opts = opts.replace(k, table[k]);
+        if (opts.match('{osm_base}')) {
+            if (!this.osmData.osmFile) throw new Error('*** {osm_base} is missing');
+            opts = opts.replace('{osm_base}', this.osmData.osmFile);
         }
 
-        return opts;
-    };
+        if (opts.match('{extracted_base}')) {
+            if (!this.osmData.extractedFile) throw new Error('*** {extracted_base} is missing');
+            opts = opts.replace('{extracted_base}', this.osmData.extractedFile);
+        }
 
-    this.setupOutputLog = (process, log) => {
-        if (process.logFunc) {
-            process.stdout.removeListener('data', process.logFunc);
-            process.stderr.removeListener('data', process.logFunc);
+        if (opts.match('{contracted_base}')) {
+            if (!this.osmData.contractedFile) throw new Error('*** {contracted_base} is missing');
+            opts = opts.replace('{contracted_base}', this.osmData.contractedFile);
         }
 
-        process.logFunc = (message) => { log.write(message); };
-        process.stdout.on('data', process.logFunc);
-        process.stderr.on('data', process.logFunc);
-    };
+        if (opts.match('{profile}')) {
+            opts = opts.replace('{profile}', [this.PROFILES_PATH, this.profile + '.lua'].join('/'));
+        }
 
-    this.runBin = (bin, options, env, callback) => {
-        let cmd = util.format('%s%s/%s%s%s', this.QQ, this.BIN_PATH, bin, this.EXE, this.QQ);
-        let opts = options.split(' ').filter((x) => { return x && x.length > 0; });
-        let log = fs.createWriteStream(this.scenarioLogFile, {'flags': 'a'});
-        log.write(util.format('*** running %s %s\n', cmd, options));
-        // we need to set a large maxbuffer here because we have long running processes like osrm-routed
-        // with lots of log output
-        let child = child_process.execFile(cmd, opts, {maxBuffer: 1024 * 1024 * 1000, env: env}, callback);
-        child.on('exit', function(code) {
-            log.end(util.format('*** %s exited with code %d\n', bin, code));
-        }.bind(this));
-        this.setupOutputLog(child, log);
-        return child;
+        var cmd = util.format('%s%s/%s%s%s %s 2>%s', this.QQ, this.BIN_PATH, bin, this.EXE, this.QQ, opts, this.ERROR_LOG_FILE);
+        process.chdir(this.TEST_FOLDER);
+        exec(cmd, (err, stdout, stderr) => {
+            this.stdout = stdout.toString();
+            fs.readFile(this.ERROR_LOG_FILE, (e, data) => {
+                this.stderr = data ? data.toString() : '';
+                this.exitCode = err && err.code || 0;
+                process.chdir('../');
+                callback(err, stdout, stderr);
+            });
+        });
     };
 };
diff --git a/features/support/shared_steps.js b/features/support/shared_steps.js
index 0252cd47b71..40c0a653061 100644
--- a/features/support/shared_steps.js
+++ b/features/support/shared_steps.js
@@ -93,7 +93,7 @@ module.exports = function () {
                             if (headers.has('distance')) {
                                 if (row.distance.length) {
                                     if (!row.distance.match(/\d+m/))
-                                        return cb(new Error('*** Distance must be specified in meters. (ex: 250m)'));
+                                        throw new Error('*** Distance must be specified in meters. (ex: 250m)');
                                     got.distance = instructions ? util.format('%dm', distance) : '';
                                 } else {
                                     got.distance = '';
@@ -102,7 +102,7 @@ module.exports = function () {
 
                             if (headers.has('time')) {
                                 if (!row.time.match(/\d+s/))
-                                    return cb(new Error('*** Time must be specied in seconds. (ex: 60s)'));
+                                    throw new Error('*** Time must be specied in seconds. (ex: 60s)');
                                 got.time = instructions ? util.format('%ds', time) : '';
                             }
 
@@ -113,7 +113,7 @@ module.exports = function () {
                             if (headers.has('speed')) {
                                 if (row.speed !== '' && instructions) {
                                     if (!row.speed.match(/\d+ km\/h/))
-                                        cb(new Error('*** Speed must be specied in km/h. (ex: 50 km/h)'));
+                                        throw new Error('*** Speed must be specied in km/h. (ex: 50 km/h)');
                                     var speed = time > 0 ? Math.round(3.6*distance/time) : null;
                                     got.speed = util.format('%d km/h', speed);
                                 } else {
@@ -139,12 +139,20 @@ module.exports = function () {
                             putValue('destinations', destinations);
                         }
 
+                        var ok = true;
+
                         for (var key in row) {
                             if (this.FuzzyMatch.match(got[key], row[key])) {
                                 got[key] = row[key];
+                            } else {
+                                ok = false;
                             }
                         }
 
+                        if (!ok) {
+                            this.logFail(row, got, { route: { query: this.query, response: res }});
+                        }
+
                         cb(null, got);
                     } else {
                         cb(new Error('request failed to return valid body'));
@@ -181,11 +189,11 @@ module.exports = function () {
 
                     if (row.from && row.to) {
                         var fromNode = this.findNodeByName(row.from);
-                        if (!fromNode) return cb(new Error(util.format('*** unknown from-node "%s"'), row.from));
+                        if (!fromNode) throw new Error(util.format('*** unknown from-node "%s"'), row.from);
                         waypoints.push(fromNode);
 
                         var toNode = this.findNodeByName(row.to);
-                        if (!toNode) return cb(new Error(util.format('*** unknown to-node "%s"'), row.to));
+                        if (!toNode) throw new Error(util.format('*** unknown to-node "%s"'), row.to);
                         waypoints.push(toNode);
 
                         got.from = row.from;
@@ -194,13 +202,13 @@ module.exports = function () {
                     } else if (row.waypoints) {
                         row.waypoints.split(',').forEach((n) => {
                             var node = this.findNodeByName(n.trim());
-                            if (!node) return cb(new Error('*** unknown waypoint node "%s"', n.trim()));
+                            if (!node) throw new Error('*** unknown waypoint node "%s"', n.trim());
                             waypoints.push(node);
                         });
                         got.waypoints = row.waypoints;
                         this.requestRoute(waypoints, bearings, params, afterRequest);
                     } else {
-                        return cb(new Error('*** no waypoints'));
+                        throw new Error('*** no waypoints');
                     }
                 }
             };
diff --git a/features/support/table_patch.js b/features/support/table_patch.js
new file mode 100644
index 00000000000..16ffebb8c07
--- /dev/null
+++ b/features/support/table_patch.js
@@ -0,0 +1,11 @@
+var DifferentError = require('./exception_classes').TableDiffError;
+
+module.exports = function () {
+    this.diffTables = (expected, actual, options, callback) => {
+        // this is a temp workaround while waiting for https://github.com/cucumber/cucumber-js/issues/534
+
+        var error = new DifferentError(expected, actual);
+
+        return callback(error.string);
+    };
+};
diff --git a/features/testbot/bad.feature b/features/testbot/bad.feature
index aeddc95d19e..18ee5f862df 100644
--- a/features/testbot/bad.feature
+++ b/features/testbot/bad.feature
@@ -11,8 +11,8 @@ Feature: Handle bad data in a graceful manner
         Given the ways
             | nodes |
 
-        When I try to run "osrm-extract {osm_file} --profile {profile_file}"
-        Then it should exit with an error
+        When the data has been contracted
+        Then "osrm-extract" should return code 1
 
     Scenario: Only dead-end oneways
         Given the node map
diff --git a/features/testbot/bugs.feature b/features/testbot/bugs.feature
new file mode 100644
index 00000000000..26be28aa6ff
--- /dev/null
+++ b/features/testbot/bugs.feature
@@ -0,0 +1,5 @@
+@routing @testbot @bug
+Feature: Known bugs
+
+    Background:
+        Given the profile "testbot"
diff --git a/features/testbot/matching.feature b/features/testbot/matching.feature
index ead4b6a7461..9c7faa01f40 100644
--- a/features/testbot/matching.feature
+++ b/features/testbot/matching.feature
@@ -124,7 +124,7 @@ Feature: Basic Map Matching
         1,2,36
         """
 
-        And the contract extra arguments "--segment-speed-file {speeds_file}"
+        And the contract extra arguments "--segment-speed-file speeds.csv"
 
         When I match I should get
             | trace | matchings | annotation                                                                                     |
diff --git a/features/testbot/traffic_turn_penalties.feature b/features/testbot/traffic_turn_penalties.feature
index 7ee9424ab54..a8f82e0dc63 100644
--- a/features/testbot/traffic_turn_penalties.feature
+++ b/features/testbot/traffic_turn_penalties.feature
@@ -26,7 +26,7 @@ Feature: Traffic - turn penalties applied to turn onto which a phantom node snap
             1,2,5,0,comment
             3,4,7,-20
             """
-        And the contract extra arguments "--turn-penalty-file {penalties_file}"
+        And the contract extra arguments "--turn-penalty-file penalties.csv"
         When I route I should get
             | from | to | route    | speed   | time    |
             | a    | e  | ab,be,be | 36 km/h | 40s +-1 |
diff --git a/features/testbot/via.feature b/features/testbot/via.feature
index 62699b8f07b..fcd9669025f 100644
--- a/features/testbot/via.feature
+++ b/features/testbot/via.feature
@@ -173,7 +173,7 @@ Feature: Via points
             | c,d,a     | abc,bd,bd,bd,abc,abc |
 
     # See issue #2349
-    @todo
+    @bug @todo
     Scenario: Via point at a dead end with oneway
         Given the node map
             | a | b | c |
diff --git a/package.json b/package.json
index c78842fa42c..87d21ce8778 100644
--- a/package.json
+++ b/package.json
@@ -7,11 +7,9 @@
     "chalk": "^1.1.3",
     "cucumber": "^1.2.1",
     "d3-queue": "^2.0.3",
-    "mkdirp": "^0.5.1",
     "node-timeout": "0.0.4",
     "polyline": "^0.2.0",
     "request": "^2.69.0",
-    "rimraf": "^2.5.4",
     "xmlbuilder": "^4.2.1"
   },
   "bin": {
diff --git a/profiles/rasterbot.lua b/profiles/rasterbot.lua
index bc2b2b247b9..03ff8f269dc 100644
--- a/profiles/rasterbot.lua
+++ b/profiles/rasterbot.lua
@@ -21,12 +21,8 @@ function way_function (way, result)
 end
 
 function source_function ()
-  local path = os.getenv('OSRM_RASTER_SOURCE')
-  if not path then
-    path = "rastersource.asc"
-  end
   raster_source = sources:load(
-    path,
+    "../test/rastersource.asc",
     0,    -- lon_min
     0.1,  -- lon_max
     0,    -- lat_min
diff --git a/profiles/rasterbotinterp.lua b/profiles/rasterbotinterp.lua
index f81e6e2f5a5..8266b07c495 100644
--- a/profiles/rasterbotinterp.lua
+++ b/profiles/rasterbotinterp.lua
@@ -21,12 +21,8 @@ function way_function (way, result)
 end
 
 function source_function ()
-  local path = os.getenv('OSRM_RASTER_SOURCE')
-  if not path then
-    path = "rastersource.asc"
-  end
   raster_source = sources:load(
-    path,
+    "../test/rastersource.asc",
     0,    -- lon_min
     0.1,  -- lon_max
     0,    -- lat_min