Rust support (#623)

.eslintignore

@@ -12,6 +12,9 @@
 /test/integration/hmr-dynamic/index.js
 /test/integration/wasm-async/index.js
 /test/integration/wasm-dynamic/index.js
+/test/integration/rust/index.js
+/test/integration/rust-deps/index.js
+/test/integration/rust-cargo/src/index.js
 
 # Generated by the build
 lib

.gitignore

@@ -11,4 +11,6 @@ lib
 !test/**/node_modules
 .vscode/
 .idea/
-*.min.js
+*.min.js
+test/integration/**/target
+test/integration/**/Cargo.lock

.travis.yml

@@ -3,8 +3,13 @@ node_js:
   # TODO: Run Babel on tests so that async-await works in Node 6
   # - '6'
   - '8'
-cache: yarn
-script: 
+cache:
+  yarn: true
+  cargo: true
+before_install:
+  - curl https://sh.rustup.rs -sSf | sh -s -- -y
+  - export PATH=/home/travis/.cargo/bin:$PATH
+script:
   - yarn test-ci
   - yarn lint
 sudo: false

appveyor.yml

@@ -9,6 +9,14 @@ install:
   # install modules
   - yarn install
 
+  # Install Rust and Cargo
+  # (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml)
+  - curl -sSf -o rustup-init.exe https://win.rustup.rs
+  - rustup-init.exe -y
+  - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+  - rustc -Vv
+  - cargo -V
+
 # Post-install test scripts.
 test_script:
   # Output useful info for debugging.
@@ -21,6 +29,7 @@ test_script:
 
 cache:
  - "%LOCALAPPDATA%\\Yarn"
+ - C:\Users\appveyor\.cargo
 
 # Don't actually build.
 build: off

package.json

@@ -17,6 +17,7 @@
     "browser-resolve": "^1.11.2",
     "chalk": "^2.1.0",
     "chokidar": "^1.7.0",
+    "command-exists": "^1.2.2",
     "commander": "^2.11.0",
     "cross-spawn": "^5.1.0",
     "cssnano": "^3.10.0",
@@ -40,6 +41,8 @@
     "sanitize-filename": "^1.6.1",
     "serve-static": "^1.12.4",
     "source-map": "0.6.1",
+    "toml": "^2.3.3",
+    "tomlify-j0.4": "^3.0.0",
     "uglify-es": "^3.2.1",
     "v8-compile-cache": "^1.1.0",
     "worker-farm": "^1.4.1",

src/Asset.js

@@ -60,7 +60,7 @@ class Asset {
 
     if (this.contents && this.mightHaveDependencies()) {
       await this.parseIfNeeded();
-      this.collectDependencies();
+      await this.collectDependencies();
     }
   }
 

src/Parser.js

@@ -31,6 +31,7 @@ class Parser {
     this.registerExtension('scss', './assets/SASSAsset');
 
     this.registerExtension('html', './assets/HTMLAsset');
+    this.registerExtension('rs', './assets/RustAsset');
 
     let extensions = options.extensions || {};
     for (let ext in extensions) {

src/WorkerFarm.js

@@ -63,18 +63,18 @@ class WorkerFarm extends Farm {
     // While we're waiting, just run on the main thread.
     // This significantly speeds up startup time.
     if (this.started && this.warmWorkers >= this.activeChildren) {
-      return this.remoteWorker.run(...args);
+      return this.remoteWorker.run(...args, false);
     } else {
       // Workers have started, but are not warmed up yet.
       // Send the job to a remote worker in the background,
       // but use the result from the local worker - it will be faster.
       if (this.started) {
-        this.remoteWorker.run(...args).then(() => {
+        this.remoteWorker.run(...args, true).then(() => {
           this.warmWorkers++;
         });
       }
 
-      return this.localWorker.run(...args);
+      return this.localWorker.run(...args, false);
     }
   }
 

src/assets/RustAsset.js

@@ -0,0 +1,213 @@
+const path = require('path');
+const commandExists = require('command-exists');
+const childProcess = require('child_process');
+const promisify = require('../utils/promisify');
+const exec = promisify(childProcess.execFile);
+const tomlify = require('tomlify-j0.4');
+const fs = require('../utils/fs');
+const Asset = require('../Asset');
+const config = require('../utils/config');
+const pipeSpawn = require('../utils/pipeSpawn');
+const md5 = require('../utils/md5');
+
+const RUST_TARGET = 'wasm32-unknown-unknown';
+const MAIN_FILES = ['src/lib.rs', 'src/main.rs'];
+
+// Track installation status so we don't need to check more than once
+let rustInstalled = false;
+let wasmGCInstalled = false;
+
+class RustAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'wasm';
+  }
+
+  process() {
+    // We don't want to process this asset if the worker is in a warm up phase
+    // since the asset will also be processed by the main process, which
+    // may cause errors since rust writes to the filesystem.
+    if (this.options.isWarmUp) {
+      return;
+    }
+
+    return super.process();
+  }
+
+  async parse() {
+    // Install rust toolchain and target if needed
+    await this.installRust();
+
+    // See if there is a Cargo config in the project
+    let cargoConfig = await this.getConfig(['Cargo.toml']);
+    let cargoDir;
+    let isMainFile = false;
+
+    if (cargoConfig) {
+      const mainFiles = MAIN_FILES.slice();
+      if (cargoConfig.lib && cargoConfig.lib.path) {
+        mainFiles.push(cargoConfig.lib.path);
+      }
+
+      cargoDir = path.dirname(await config.resolve(this.name, ['Cargo.toml']));
+      isMainFile = mainFiles.some(
+        file => path.join(cargoDir, file) === this.name
+      );
+    }
+
+    // If this is the main file of a Cargo build, use the cargo command to compile.
+    // Otherwise, use rustc directly.
+    if (isMainFile) {
+      await this.cargoBuild(cargoConfig, cargoDir);
+    } else {
+      await this.rustcBuild();
+    }
+
+    // If this is a prod build, use wasm-gc to remove unused code
+    if (this.options.minify) {
+      await this.installWasmGC();
+      await exec('wasm-gc', [this.wasmPath, this.wasmPath]);
+    }
+  }
+
+  async installRust() {
+    if (rustInstalled) {
+      return;
+    }
+
+    // Check for rustup
+    try {
+      await commandExists('rustup');
+    } catch (e) {
+      throw new Error(
+        "Rust isn't installed. Visit https://www.rustup.rs/ for more info"
+      );
+    }
+
+    // Ensure nightly toolchain is installed
+    let [stdout] = await exec('rustup', ['show']);
+    if (!stdout.includes('nightly')) {
+      await pipeSpawn('rustup', ['update']);
+      await pipeSpawn('rustup', ['toolchain', 'install', 'nightly']);
+    }
+
+    // Ensure wasm target is installed
+    [stdout] = await exec('rustup', [
+      'target',
+      'list',
+      '--toolchain',
+      'nightly'
+    ]);
+    if (!stdout.includes(RUST_TARGET + ' (installed)')) {
+      await pipeSpawn('rustup', [
+        'target',
+        'add',
+        'wasm32-unknown-unknown',
+        '--toolchain',
+        'nightly'
+      ]);
+    }
+
+    rustInstalled = true;
+  }
+
+  async installWasmGC() {
+    if (wasmGCInstalled) {
+      return;
+    }
+
+    try {
+      await commandExists('wasm-gc');
+    } catch (e) {
+      await pipeSpawn('cargo', [
+        'install',
+        '--git',
+        'https://github.com/alexcrichton/wasm-gc'
+      ]);
+    }
+
+    wasmGCInstalled = true;
+  }
+
+  async cargoBuild(cargoConfig, cargoDir) {
+    // Ensure the cargo config has cdylib as the crate-type
+    if (!cargoConfig.lib) {
+      cargoConfig.lib = {};
+    }
+
+    if (!Array.isArray(cargoConfig.lib['crate-type'])) {
+      cargoConfig.lib['crate-type'] = [];
+    }
+
+    if (!cargoConfig.lib['crate-type'].includes('cdylib')) {
+      cargoConfig.lib['crate-type'].push('cdylib');
+      await fs.writeFile(
+        path.join(cargoDir, 'Cargo.toml'),
+        tomlify.toToml(cargoConfig)
+      );
+    }
+
+    // Run cargo
+    let args = ['+nightly', 'build', '--target', RUST_TARGET, '--release'];
+    await exec('cargo', args, {cwd: cargoDir});
+
+    // Get output file paths
+    let outDir = path.join(cargoDir, 'target', RUST_TARGET, 'release');
+    let rustName = cargoConfig.package.name;
+    this.wasmPath = path.join(outDir, rustName + '.wasm');
+    this.depsPath = path.join(outDir, rustName + '.d');
+  }
+
+  async rustcBuild() {
+    // Get output filename
+    await fs.mkdirp(this.options.cacheDir);
+    let name = md5(this.name);
+    this.wasmPath = path.join(this.options.cacheDir, name + '.wasm');
+
+    // Run rustc to compile the code
+    const args = [
+      '+nightly',
+      '--target',
+      RUST_TARGET,
+      '-O',
+      '--crate-type=cdylib',
+      this.name,
+      '-o',
+      this.wasmPath
+    ];
+    await exec('rustc', args);
+
+    // Run again to collect dependencies
+    this.depsPath = path.join(this.options.cacheDir, name + '.d');
+    await exec('rustc', [this.name, '--emit=dep-info', '-o', this.depsPath]);
+  }
+
+  async collectDependencies() {
+    // Read deps file
+    let contents = await fs.readFile(this.depsPath, 'utf8');
+    let dir = path.dirname(this.name);
+
+    let deps = contents
+      .split('\n')
+      .filter(Boolean)
+      .slice(1);
+
+    for (let dep of deps) {
+      dep = path.resolve(dir, dep.slice(0, dep.indexOf(':')));
+      if (dep !== this.name) {
+        this.addDependency(dep, {includedInParent: true});
+      }
+    }
+  }
+
+  async generate() {
+    return {
+      wasm: {
+        path: this.wasmPath, // pass output path to RawPackager
+        mtime: Date.now() // force re-bundling since otherwise the hash would never change
+      }
+    };
+  }
+}
+
+module.exports = RustAsset;

src/packagers/RawPackager.js

@@ -18,8 +18,11 @@ class RawPackager extends Packager {
       );
     }
 
-    let contents =
-      asset.generated[asset.type] || (await fs.readFile(asset.name));
+    let contents = asset.generated[asset.type];
+    if (!contents || (contents && contents.path)) {
+      contents = await fs.readFile(contents ? contents.path : asset.name);
+    }
+
     await fs.writeFile(name, contents);
   }
 

src/utils/config.js

@@ -1,6 +1,10 @@
 const fs = require('./fs');
 const path = require('path');
-const json5 = require('json5');
+
+const PARSERS = {
+  json: require('json5').parse,
+  toml: require('toml').parse
+};
 
 const existsCache = new Map();
 
@@ -30,12 +34,14 @@ async function load(filepath, filenames, root = path.parse(filepath).root) {
   let configFile = await resolve(filepath, filenames, root);
   if (configFile) {
     try {
-      if (path.extname(configFile) === '.js') {
+      let extname = path.extname(configFile).slice(1);
+      if (extname === 'js') {
         return require(configFile);
       }
 
       let configStream = await fs.readFile(configFile);
-      return json5.parse(configStream.toString());
+      let parse = PARSERS[extname] || PARSERS.json;
+      return parse(configStream.toString());
     } catch (err) {
       if (err.code === 'MODULE_NOT_FOUND' || err.code === 'ENOENT') {
         existsCache.delete(configFile);

src/utils/objectHash.js

@@ -1,10 +1,17 @@
 const crypto = require('crypto');
 
-module.exports = function(object) {
+function objectHash(object) {
   let hash = crypto.createHash('md5');
   for (let key of Object.keys(object).sort()) {
-    hash.update(key + object[key]);
+    let val = object[key];
+    if (typeof val === 'object' && val) {
+      hash.update(key + objectHash(val));
+    } else {
+      hash.update(key + val);
+    }
   }
 
   return hash.digest('hex');
-};
+}
+
+module.exports = objectHash;