diff --git a/README.md b/README.md index 5542b34b9..3b51a989f 100644 --- a/README.md +++ b/README.md @@ -214,10 +214,17 @@ This way you may even avoid creating `pkg` config for your project. ## Native addons -Native addons (`.node` files) use is supported, but packaging -`.node` files inside the executable is not resolved yet. You have -to deploy native addons used by your project to the same directory -as the executable. +Native addons (`.node` files) use is supported. When `pkg` encounters +a `.node` file in a `require` call, it will package this like an asset. +In some cases (like with the `bindings` package), the module path is generated +dynamicaly and `pkg` won't be able to detect it. In this case, you should +add the `.node` file directly in the `assets` field in `package.json`. + +The way Node.js requires native addon is different from a classic JS +file. It needs to have a file on disk to load it, but `pkg` only generates +one file. To circumvent this, `pkg` will create a temporary file on the +disk. These files will stay on the disk after the process has exited +and will be used again on the next process launch. When a package, that contains a native module, is being installed, the native module is compiled against current system-wide Node.js diff --git a/lib/packer.js b/lib/packer.js index 86cd10e4e..2305f125f 100644 --- a/lib/packer.js +++ b/lib/packer.js @@ -2,7 +2,7 @@ import { STORE_BLOB, STORE_CONTENT, STORE_LINKS, - STORE_STAT, isDotJS, isDotJSON, isDotNODE + STORE_STAT, isDotJS, isDotJSON } from '../prelude/common.js'; import { log, wasReported } from './log.js'; @@ -34,18 +34,13 @@ function hasAnyStore (record) { export default function ({ records, entrypoint, bytecode }) { const stripes = []; - for (const snap in records) { const record = records[snap]; const { file } = record; if (!hasAnyStore(record)) continue; assert(record[STORE_STAT], 'packer: no STORE_STAT'); - if (isDotNODE(file)) { - continue; - } else { - assert(record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]); - } + assert(record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]); if (record[STORE_BLOB] && !bytecode) { delete record[STORE_BLOB]; diff --git a/lib/walker.js b/lib/walker.js index 4a0f87be6..5d9f68cb9 100644 --- a/lib/walker.js +++ b/lib/walker.js @@ -522,24 +522,11 @@ class Walker { store: STORE_STAT }); - if (isDotNODE(record.file)) { - // provide explicit deployFiles to override - // native addon deployment place. see 'sharp' - if (!marker.hasDeployFiles) { - log.warn('Cannot include addon %1 into executable.', [ - 'The addon must be distributed with executable as %2.', - '%1: ' + record.file, - '%2: path-to-executable/' + path.basename(record.file) ]); - } - return; // discard - } - const derivatives1 = []; await this.stepActivate(marker, derivatives1); await this.stepDerivatives(record, marker, derivatives1); - if (store === STORE_BLOB) { - if (unlikelyJavascript(record.file)) { + if (unlikelyJavascript(record.file) || isDotNODE(record.file)) { this.append({ file: record.file, marker, diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 8f51e62f6..62dccfe52 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -1580,3 +1580,65 @@ function payloadFileSync (pointer) { }); } }()); + +// ///////////////////////////////////////////////////////////////// +// PATCH PROCESS /////////////////////////////////////////////////// +// ///////////////////////////////////////////////////////////////// + +(function () { + const fs = require('fs'); + var ancestor = {}; + ancestor.dlopen = process.dlopen; + + process.dlopen = function () { + const args = cloneArgs(arguments); + const modulePath = args[1]; + const moduleDirname = require('path').dirname(modulePath); + if (insideSnapshot(modulePath)) { + // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path + // we need to write the file somewhere on disk first and then load it + const moduleContent = fs.readFileSync(modulePath); + const moduleBaseName = require('path').basename(modulePath); + const hash = require('crypto').createHash('sha256').update(moduleContent).digest('hex'); + const tmpModulePath = `${require('os').tmpdir()}/${hash}_${moduleBaseName}`; + try { + fs.statSync(tmpModulePath); + } catch (e) { + // Most likely this means the module is not on disk yet + fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 }); + } + args[1] = tmpModulePath; + } + + const unknownModuleErrorRegex = /([^:]+): cannot open shared object file: No such file or directory/; + const tryImporting = function (previousErrorMessage) { + try { + const res = ancestor.dlopen.apply(process, args); + return res; + } catch (e) { + if (e.message === previousErrorMessage) { + // we already tried to fix this and it didn't work, give up + throw e; + } + if (e.message.match(unknownModuleErrorRegex)) { + // some modules are packaged with dynamic linking and needs to open other files that should be in + // the same directory, in this case, we write this file in the same /tmp directory and try to + // import the module again + const moduleName = e.message.match(unknownModuleErrorRegex)[1]; + const importModulePath = `${moduleDirname}/${moduleName}`; + const moduleContent = fs.readFileSync(importModulePath); + const moduleBaseName = require('path').basename(importModulePath); + const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`; + try { + fs.statSync(tmpModulePath); + } catch (err) { + fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 }); + } + return tryImporting(e.message); + } + throw e; + } + }; + tryImporting(); + }; +}()); diff --git a/test/test-50-cannot-include-addon/main.js b/test/test-50-can-include-addon/main.js similarity index 83% rename from test/test-50-cannot-include-addon/main.js rename to test/test-50-can-include-addon/main.js index f59d21738..5a13acd27 100644 --- a/test/test-50-cannot-include-addon/main.js +++ b/test/test-50-can-include-addon/main.js @@ -26,6 +26,6 @@ right = utils.pkg.sync([ assert(right.indexOf('\x1B\x5B') < 0, 'colors detected'); right = right.replace(/\\/g, '/'); -assert(right.indexOf('test-50-cannot-include-addon/time.node') >= 0); -assert(right.indexOf('path-to-executable/time.node') >= 0); +assert(right.indexOf('test-50-can-include-addon/time.node') === -1); +assert(right.indexOf('path-to-executable/time.node') === -1); utils.vacuum.sync(output); diff --git a/test/test-50-cannot-include-addon/test-x-index.js b/test/test-50-can-include-addon/test-x-index.js similarity index 100% rename from test/test-50-cannot-include-addon/test-x-index.js rename to test/test-50-can-include-addon/test-x-index.js diff --git a/test/test-50-can-include-addon/time.node b/test/test-50-can-include-addon/time.node new file mode 100644 index 000000000..37a464842 --- /dev/null +++ b/test/test-50-can-include-addon/time.node @@ -0,0 +1 @@ +module.exports = 'test'; diff --git a/test/test-50-cannot-include-addon/time.node b/test/test-50-cannot-include-addon/time.node deleted file mode 100644 index aa43fbb13..000000000 --- a/test/test-50-cannot-include-addon/time.node +++ /dev/null @@ -1 +0,0 @@ -module.exports = require('path').basename(__filename); diff --git a/test/test-50-native-addon-2/node_modules/dependency/time.node b/test/test-50-native-addon-2/node_modules/dependency/time.node index aa43fbb13..37a464842 100644 --- a/test/test-50-native-addon-2/node_modules/dependency/time.node +++ b/test/test-50-native-addon-2/node_modules/dependency/time.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'test'; diff --git a/test/test-50-native-addon-3/lib/community/time-y.node b/test/test-50-native-addon-3/lib/community/time-y.node index aa43fbb13..7a4cb5f89 100644 --- a/test/test-50-native-addon-3/lib/community/time-y.node +++ b/test/test-50-native-addon-3/lib/community/time-y.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'time-y'; diff --git a/test/test-50-native-addon-3/lib/enterprise/time-z.node b/test/test-50-native-addon-3/lib/enterprise/time-z.node index aa43fbb13..1e395f40d 100644 --- a/test/test-50-native-addon-3/lib/enterprise/time-z.node +++ b/test/test-50-native-addon-3/lib/enterprise/time-z.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'time-z'; diff --git a/test/test-50-native-addon-3/lib/time-x.node b/test/test-50-native-addon-3/lib/time-x.node index aa43fbb13..37a464842 100644 --- a/test/test-50-native-addon-3/lib/time-x.node +++ b/test/test-50-native-addon-3/lib/time-x.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'test'; diff --git a/test/test-50-native-addon-3/node_modules/dependency/time-d.node b/test/test-50-native-addon-3/node_modules/dependency/time-d.node index aa43fbb13..5b8b4f773 100644 --- a/test/test-50-native-addon-3/node_modules/dependency/time-d.node +++ b/test/test-50-native-addon-3/node_modules/dependency/time-d.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'time-d'; diff --git a/test/test-50-native-addon-4/lib/time.node b/test/test-50-native-addon-4/lib/time.node new file mode 100644 index 000000000..37a464842 --- /dev/null +++ b/test/test-50-native-addon-4/lib/time.node @@ -0,0 +1 @@ +module.exports = 'test'; diff --git a/test/test-50-native-addon-4/main.js b/test/test-50-native-addon-4/main.js new file mode 100644 index 000000000..30f878589 --- /dev/null +++ b/test/test-50-native-addon-4/main.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +'use strict'; + +const path = require('path'); +const assert = require('assert'); +const utils = require('../utils.js'); + +assert(!module.parent); +assert(__dirname === process.cwd()); + +const host = 'node' + process.version.match(/^v(\d+)/)[1]; +const target = process.argv[2] || host; +const input = './test-x-index.js'; +const output = './run-time/test-output.exe'; + +let left, right; +utils.mkdirp.sync(path.dirname(output)); + +left = utils.spawn.sync( + 'node', [ path.basename(input) ], + { cwd: path.dirname(input) } +); + +utils.pkg.sync([ + '--target', target, + '--output', output, input +]); + +right = utils.spawn.sync( + './' + path.basename(output), [], + { cwd: path.dirname(output) } +); + +assert.equal(left, right); +utils.vacuum.sync(path.dirname(output)); diff --git a/test/test-50-native-addon-4/test-x-index.js b/test/test-50-native-addon-4/test-x-index.js new file mode 100644 index 000000000..b66cc79e2 --- /dev/null +++ b/test/test-50-native-addon-4/test-x-index.js @@ -0,0 +1,10 @@ +/* eslint-disable no-underscore-dangle */ + +'use strict'; + +var fs = require('fs'); +var path = require('path'); +var Module = require('module'); +Module._extensions['.node'] = Module._extensions['.js']; +console.log(fs.existsSync(path.join(__dirname, 'lib/time.node'))); +console.log(require('./lib/time.node')); diff --git a/test/test-50-native-addon/lib/time.node b/test/test-50-native-addon/lib/time.node index aa43fbb13..37a464842 100644 --- a/test/test-50-native-addon/lib/time.node +++ b/test/test-50-native-addon/lib/time.node @@ -1 +1 @@ -module.exports = require('path').basename(__filename); +module.exports = 'test';