diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9823984..0000000 --- a/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -/examples -/output/ -/apt-cacher-ng/ -/.git diff --git a/.gitignore b/.gitignore index 43f6e73..239ecff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -/output/ -/apt-cacher-ng/ node_modules +yarn.lock diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..9cf9495 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d11655a..0000000 --- a/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG TARGET - -FROM dockcross/${TARGET} - -RUN apt-get -y update && \ - apt-get -y --no-install-recommends install \ - git curl gnupg apt-transport-https \ - && \ - curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ - echo "deb https://deb.nodesource.com/node_10.x buster main" | tee /etc/apt/sources.list.d/nodesource.list && \ - echo "deb-src https://deb.nodesource.com/node_10.x buster main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ - apt-get -y update && \ - apt-get -y install nodejs && \ - rm -rf /var/lib/apt/lists/* - -ENV STRIP ${CROSS_ROOT}/bin/${CROSS_TRIPLE}-strip - -COPY ./build-in-docker /app/ - -VOLUME ["/app/input"] diff --git a/README.md b/README.md index 735d6b9..3d3d626 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,66 @@ # prebuildify-cross -cross-compile [prebuild](https://github.com/mafintosh/prebuildify)s +**Compile prebuilds in Docker, supporting Linux (including Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and up), Alpine Linux, ARM Linux devices like the Raspberry Pi and mobile ARM devices like Android.** -## background +Runs [`prebuildify`](https://github.com/mafintosh/prebuildify) in preconfigured [`prebuild/docker-images`](https://github.com/prebuild/docker-images) containers to compile and name prebuilds for a certain platform. This means you don't have to worry about GCC flags, environment variables or system dependencies. In addition, `prebuildify-cross` copies only npm package files to Docker (following the rules of `.npmignore` and `files`) and mounts `node_modules` removing the need for a repeated `npm install`. -i want to build native modules for [Scuttlebutt](https://scuttlebutt.nz) pubs, to use on ARM Linux devices like the Raspberry Pi. meanwhile, i also want to support mobile ARM devices like Android. +## Install -## how does it work? - -`prebuildify-cross` will: - -- build a Docker image for your intended target, based on the respective [`dockcross`](https://github.com/dockcross/dockcross) image -- create a Docker container with your input (default is `.`) mounted inside -- run `npm install --ignore-scripts` and `npm run prebuild` -- copy out `./prebuilds` from the container to your output (default is `./prebuilds`) - -## supported targets - -- `prebuildify-cross --platform=linux --arch=x32` -- `prebuildify-cross --platform=linux --arch=x64` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=5` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=6` -- `prebuildify-cross --platform=linux --arch=arm --arm-version=7` -- `prebuildify-cross --platform=linux --arch=arm64` -- `prebuildify-cross --platform=android --arch=arm --arm-version=7` -- `prebuildify-cross --platform=android --arch=arm64` - -## usage - -(note: `prebuildify-cross` depends on having Docker installed.) - -in the module you want to cross-compile prebuilds, - -ensure you have an npm script `prebuild`, like: +Depends on having Docker installed, as well as `prebuildify` and `node-gyp`: ``` -{ - "scripts": { - "prebuild": "prebuildify --all --strip" - } -} +npm install --save-dev prebuildify node-gyp prebuildify-cross ``` -then install `prebuildify-cross` as a dev-dependency: +## Usage + +The `prebuildify-cross` cli forwards all command line arguments to `prebuildify`, but adds an `--image` or `-i` argument. For example, the following command will invoke `prebuildify -t 8.14.0 --napi --strip` in a CentOS container and copy the resulting prebuild to `./prebuilds`: ``` -npm install --save-dev prebuildify-cross +prebuildify-cross -i centos7-devtoolset7 -t 8.14.0 --napi --strip ``` -then add new `prebuild:cross:${TARGET}` scripts for the targets you want to support: +To build for more than one platform, multiple `--image` arguments may be passed: ``` -{ - "scripts": { - "prebuild:cross:linux-armv7": "prebuildify-cross --platform linux --arch arm --arm-version 7", - "prebuild:cross:android-armv7": "prebuildify-cross --platform android --arch arm --arm-version 7" - } -} +prebuildify-cross -i linux-armv7 -i linux-arm64 -t .. ``` -then when you want to cross-compile prebuilds, `npm run` the appropriate script. - -for the full command-line usage: +Lastly, it's possible to use your own custom image with e.g. `-i my-namespace/my-image`. -``` -Usage: - prebuildify-cross [options] +## Images - Arguments: +### `centos7-devtoolset7` - --arch : **required** architecture (supported: x32, x64, arm, arm64) - --arm-version : if using arm architecture, **required* arm version (supported: 5, 6, 7, 8) - --platform: **required** platform (supported: linux, android) +Compile in CentOS 7, as a better alternative to (commonly) Ubuntu 16.04 on Travis. Makes the prebuild compatible with Debian 8, Ubuntu 14.04, RHEL 7, CentOS 7 and other Linux flavors with an old glibc. - --input : _optional_ directory of input image spec (default: cwd) - --output : _optional_ directory to output build results (default: prebuilds) +> The neat thing about this is that you get to compile with gcc 7 but glibc 2.17, so binaries are compatible for \[among others] Ubuntu 14.04 and Debian 8. +> +> The RHEL folks put in a ton of work to make the devtoolsets work on their older base systems (libc mainly), which involves shipping a delta library that contains the new stuff that can be statically linked in where it's used. We use this method for building Node binary releases. +> +> \-- [**@rvagg**](https://github.com/rvagg) ([prebuild/docker-images#8](https://github.com/prebuild/docker-images/pull/8)) - Flags: +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor to set it apart from Alpine prebuilds, e.g. `linux-x64/node.libc.node`. - -h, --help: show this usage +### `alpine` - Examples: +Compile in Alpine, which uses musl instead of glibc and therefore can't run regular linux prebuilds. Worse, it sometimes does successfully _load_ such a prebuild during `npm install` - which prevents a compilation fallback from kicking in - and then segfaults at runtime. You can fix this situation in two ways: by shipping an `alpine` prebuild and/or by shipping a `centos7-devtoolset7` prebuild, because the latter will be skipped in Alpine thanks to the `libc` tag. - prebuildify-cross --platform linux --arch x64 +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the libc flavor, e.g. `linux-x64/node.musl.node`. - prebuildify-cross --platform linux --arch arm --arm-version 7 +### `linux-armv7` and `linux-arm64` - prebuildify-cross --platform linux --arch arm64 +Cross-compile for Linux ARM. These images thinly wrap [`dockcross`](https://github.com/dockcross/dockcross) images. - prebuildify-cross --platform android --arch arm --arm-version 7 +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the armv version (7 or 8, respectively). - prebuildify-cross --platform android --arch arm64 -``` +### `android-armv7` and `android-arm64` -you can also pass in environment variables instead of command-line arguments, e.g. +Cross-compile for Android ARM. These images thinly wrap [`dockcross`](https://github.com/dockcross/dockcross) images. -```shell -PLATFORM=linux ARCH=arm ARM_VERSION=7 prebuildify-cross -``` +By default the prebuild will be [tagged](https://github.com/prebuild/prebuildify#options) with the armv version (7 or 8, respectively). -## references +## References - [Debian multiarch tuples](https://wiki.debian.org/Multiarch/Tuples) - [Rust support tuples](https://forge.rust-lang.org/platform-support.html) @@ -107,8 +68,7 @@ PLATFORM=linux ARCH=arm ARM_VERSION=7 prebuildify-cross - [Arm options](https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html) - [Rust cross-compiling tutorial](https://github.com/japaric/rust-cross) - [Rust cross-compilation tool](https://github.com/rust-embedded/cross) -- [`leveldown` cross-compilation discussion](https://github.com/Level/leveldown/pull/572) -## license +## License GPL-3.0 diff --git a/build b/build deleted file mode 100755 index 1e1389e..0000000 --- a/build +++ /dev/null @@ -1,256 +0,0 @@ -#!/bin/bash - -CWD=$(pwd) -# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within -DIRNAME="$(dirname "$(readlink -f "$0")")" - -# with help from https://stackoverflow.com/a/29754866, - -# saner programming env: these switches turn some bugs into errors -set -o errexit -o pipefail -o noclobber -o nounset - -DOCKER="docker" -set +e -if ! $DOCKER ps >/dev/null; then - echo "error connecting to docker:" - $DOCKER ps - exit 1 -fi -set -e - -! getopt --test > /dev/null -if [[ ${PIPESTATUS[0]} -ne 4 ]]; then - echo "GNU's enhanced getopt is required to run this script" - echo "You can usually find this in the util-linux package" - echo "On MacOS/OS X see homebrew's package: http://brewformulas.org/Gnu-getopt" - echo "For anyone else, build from source: http://frodo.looijaard.name/project/getopt" - exit 1 -fi - -OPTIONS=hi:o: -LONGOPTS=help,input:,output:,arch:,arm-version:,platform: - -# -use ! and PIPESTATUS to get exit code with errexit set -# -temporarily store output to be able to check for errors -# -activate quoting/enhanced mode (e.g. by writing out “--options”) -# -pass arguments only via -- "$@" to separate them correctly -! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@") -if [[ ${PIPESTATUS[0]} -ne 0 ]]; then - # e.g. return value is 1 - # then getopt has complained about wrong arguments to stdout - exit 2 -fi -# read getopt’s output this way to handle the quoting right: -eval set -- "$PARSED" - -HELP=n -ARCH=${ARCH:-} ARM_VERSION=${ARM_VERSION:-} PLATFORM=${PLATFORM:-} -ARGS="" -INPUT="${CWD}" OUTPUT=prebuilds -while true; do - case "$1" in - -h|--help) - HELP=y - shift - ;; - -i|--input) - INPUT="$2" - shift 2 - ;; - -o|--output) - OUTPUT="$2" - shift 2 - ;; - --arch) - ARCH="$2" - shift 2 - ;; - --arm-version) - ARM_VERSION="$2" - shift 2 - ;; - --platform) - PLATFORM="$2" - shift 2 - ;; - --) - shift - ARGS="$@" - break - ;; - *) - echo "Programming error" - exit 3 - ;; - esac -done - -if - [ "${HELP}" == "y" ] || \ - [ "${ARCH}" == "" ] || \ - ( [ "${ARCH}" != "x32" ] && [ "${ARCH}" != "x64" ] && [ "${ARCH}" != "arm" ] && [ "${ARCH}" != "arm64" ] ) || \ - [ "${PLATFORM}" == "" ] || \ - ( [ "${PLATFORM}" != "android" ] && [ "${PLATFORM}" != "linux" ] ) || \ - [ "${ARCH}" == "arm" ] && [ "${ARM_VERSION}" == "" ] -then - cat >&2 <: **required** architecture (supported: x32, x64, arm, arm64) - --arm-version : if using arm architecture, **required* arm version (supported: 5, 6, 7, 8) - --platform: **required** platform (supported: linux, android) - - --input : _optional_ directory of input image spec (default: cwd) - --output : _optional_ directory to output build results (default: prebuilds) - - Flags: - - -h, --help: show this usage - - Examples: - - prebuildify-cross --platform linux --arch x64 - - prebuildify-cross --platform linux --arch arm --arm-version 7 - - prebuildify-cross --platform linux --arch arm64 - - prebuildify-cross --platform android --arch arm --arm-version 7 - - prebuildify-cross --platform android --arch arm64 -EOF - exit 1 -fi - -if [[ "${INPUT}" != /* ]] -then - INPUT="${CWD}/${INPUT}" -fi - -if [[ "${OUTPUT}" != /* ]] -then - OUTPUT="${CWD}/${OUTPUT}" -fi - -echo input $INPUT -echo output $OUTPUT - -if [ "${ARM_VERSION}" == "8" ] -then - ARCH=arm64 -fi - -if [ "${ARCH}" == "arm64" ] -then - ARM_VERSION=8 -fi - -function not_supported () { - if [ "${ARM_VERSION}" == "" ] - then - echo "platform=${PLATFORM}, arch=${ARCH}: not supported!" - else - echo "platform=${PLATFORM}, arch=${ARCH}, arm_version=${ARM_VERSION}: not supported!" - fi - exit 1 -} - -case "${PLATFORM}" in - "android") - case "${ARM_VERSION}" in - 7) - TARGET=android-arm - ;; - 8) - TARGET=android-arm64 - ;; - *) - not_supported - ;; - esac - ;; - "linux") - case "${ARM_VERSION}" in - 5|6|7) - TARGET="linux-armv${ARM_VERSION}" - ;; - 8) - TARGET=linux-arm64 - ;; - "") - TARGET=linux-${ARCH} - ;; - *) - not_supported - ;; - esac - ;; - *) - not_supported - ;; -esac - -echo args $ARGS - -IMAGE_NAME=${IMAGE_NAME:-prebuildify-cross} -CONTAINER_NAME=${CONTAINER_NAME:-prebuildify-cross-work-${TARGET}} - -CONTAINER_RUNNING=$($DOCKER ps --filter name="$CONTAINER_NAME" -q) -if [ "$CONTAINER_RUNNING" != "" ]; then - $DOCKER stop ${CONTAINER_NAME} > /dev/null -fi - -CONTAINER_EXISTS=$($DOCKER ps -a --filter name="$CONTAINER_NAME" -q) -if [ "$CONTAINER_EXISTS" != "" ]; then - $DOCKER rm -v "${CONTAINER_NAME}" -fi - -function cleanup { - echo 'got EXIT signal... please wait' - - CONTAINER_RUNNING=$($DOCKER ps --filter name="${CONTAINER_NAME}" -q) - if [ "${CONTAINER_RUNNING}" != "" ]; then - $DOCKER stop ${CONTAINER_NAME} > /dev/null - fi - - CONTAINER_EXISTS=$($DOCKER ps -a --filter name="${CONTAINER_NAME}" -q) - if [ "${CONTAINER_EXISTS}" != "" ]; then - $DOCKER rm -v ${CONTAINER_NAME} > /dev/null - fi -} - -trap cleanup EXIT - -echo ARCH "${ARCH}" -echo ARM_VERSION "${ARM_VERSION}" -echo PLATFORM "${PLATFORM}" -echo TARGET "${TARGET}" - -# build target image -$DOCKER build \ - --build-arg TARGET="${TARGET}" \ - -t ${IMAGE_NAME}:${TARGET} \ - ${DIRNAME} - -$DOCKER run \ - --name "${CONTAINER_NAME}" \ - -d \ - -v "${INPUT}:/app/input" \ - --env ARCH="${ARCH}" \ - --env ARM_VERSION="${ARM_VERSION}" \ - --env TARGET_PLATFORM="${PLATFORM}" \ - ${IMAGE_NAME}:${TARGET} \ - bash -e -o errexit -o pipefail -o noclobber -o nounset -c " - cd /app; ./build-in-docker ${ARGS}; - " -time $DOCKER logs "${CONTAINER_NAME}" --follow || true & -wait "$!" - -echo "copying results from output/" -$DOCKER cp "${CONTAINER_NAME}":/app/output/. "${OUTPUT}" -ls -lah "${OUTPUT}" - -echo "Done! Your prebuilds should be in ${OUTPUT}" diff --git a/build-in-docker b/build-in-docker deleted file mode 100755 index 8f1825c..0000000 --- a/build-in-docker +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -ARGUMENTS="$@" -CWD=$(pwd) - -# saner programming env: these switches turn some bugs into errors -set -o errexit -o pipefail -o noclobber -o nounset - -umask 022 - -cp -r "${CWD}/input" "${CWD}/work" - -cd "${CWD}/work" - -npm install --ignore-scripts - -PATH="$(npm bin):$PATH" prebuildify $ARGUMENTS - -cp -r "${CWD}/work/prebuilds" "${CWD}/output/" diff --git a/cli.js b/cli.js new file mode 100644 index 0000000..a84f161 --- /dev/null +++ b/cli.js @@ -0,0 +1,13 @@ +#!/usr/bin/env node +'use strict' + +const argv = process.argv.slice(2) + +const opts = require('minimist')(argv, { + string: ['image', 'cwd'], + alias: { image: 'i' } +}) + +require('.')({ argv, ...opts }, function (err) { + if (err) throw err +}) diff --git a/guest.js b/guest.js new file mode 100644 index 0000000..55d8438 --- /dev/null +++ b/guest.js @@ -0,0 +1,35 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const cp = require('child_process') +const mkdirp = require('mkdirp') + +// TODO: fix permissions of WORKDIR in prebuild images +const cwd = '/home/node/app' +const files = JSON.parse(process.env.PREBUILDIFY_CROSS_FILES) +const argv = process.argv.slice(2) + +// Copy host files to working directory +for (const file of files) { + const a = path.join('/input', file) + const b = path.join(cwd, file) + + mkdirp.sync(path.dirname(b)) + + fs.copyFileSync(a, b, fs.constants.COPYFILE_EXCL) + fs.chmodSync(b, 0o644) +} + +// Use node_modules of host to avoid a second install step +fs.symlinkSync('/input/node_modules', path.join(cwd, 'node_modules')) + +const stdio = ['ignore', 2, 2] +const res = cp.spawnSync('npx', ['prebuildify', ...argv], { cwd, stdio }) + +if (res.status) process.exit(res.status) +if (res.error) throw res.error + +// Write tarball to stdout. With this approach we don't need +// a writable volume and can avoid messing with permissions. +require('tar-fs').pack(path.join(cwd, 'prebuilds')).pipe(process.stdout) diff --git a/index.js b/index.js new file mode 100644 index 0000000..906c945 --- /dev/null +++ b/index.js @@ -0,0 +1,140 @@ +'use strict' + +const packlist = require('npm-packlist') +const tar = require('tar-fs') +const dockerPull = require('docker-pull') +const dockerRun = require('docker-run') +const logger = require('log-update') +const bytes = require('pretty-bytes') +const browserify = require('browserify') +const unixify = require('unixify') +const once = require('once') +const path = require('path') + +module.exports = function (opts, callback) { + if (typeof opts === 'function') { + callback = opts + opts = null + } + + opts = opts || {} + callback = once(callback) + + const images = [].concat(opts.image || []) + const cwd = path.resolve(opts.cwd || '.') + const files = JSON.stringify(packageFiles(cwd)) + const prebuilds = path.join(cwd, 'prebuilds') + const log = logger.create(process.stderr, { showCursor: true }) + + loop() + + function loop () { + let image = images.shift() + if (!image) return process.nextTick(callback) + if (!image.includes('/')) image = 'prebuild/' + image + + dockerPull(image) + .on('progress', progress) + .on('error', callback) + .on('end', end) + + function progress () { + if (process.env.CI) { + console.error(`> prebuildify-cross pull ${this.image}`) + return this.removeListener('progress', progress) + } + + const count = `${this.layers} layers` + const ratio = `${bytes(this.transferred)} / ${bytes(this.length)}` + + log(`> prebuildify-cross pull ${this.image}: ${count}, ${ratio}`) + } + + function end () { + run(this.image) + } + } + + function run (image) { + const argv = prebuildifyArgv(opts.argv || [], image) + + console.error('> prebuildify-cross run %s', image) + console.error('> prebuildify %s\n', argv.join(' ')) + + const child = dockerRun(image, { + entrypoint: 'node', + argv: ['-'].concat(argv), + volumes: { + // Should but can't use :ro (mafintosh/docker-run#12) + [cygwin(cwd)]: '/input' + }, + env: { + PREBUILDIFY_CROSS_FILES: files, + // Disable npm update check + NO_UPDATE_NOTIFIER: 'true' + } + }) + + child + .on('error', callback) + .on('exit', onexit) + + guestScript() + .pipe(child.stdin) + + child.stderr + .pipe(process.stderr) + + child.stdout + .pipe(tar.extract(prebuilds), { dmode: 0o755, fmode: 0o644 }) + .on('finish', loop) + .on('error', callback) + } + + function onexit (code) { + if (code) return callback(new Error('Exited with code ' + code)) + } +} + +function packageFiles (dir) { + return packlist.sync({ dir }).filter(function (fp) { + return !/^prebuilds[/\\]/i.test(fp) + }) +} + +function guestScript () { + return browserify(require.resolve('./guest.js'), { + basedir: __dirname, + node: true + }).bundle() +} + +function prebuildifyArgv (argv, image) { + argv = argv.slice() + + for (let i = 0; i < argv.length - 1; i++) { + if (/^(-i|--image)$/.test(argv[i]) && argv[i + 1][0] !== '-') { + argv.splice(i--, 2) + } + } + + // TODO: move this to the docker images? + if (/^prebuild\/(linux|android)-arm/.test(image)) argv.push('--tag-armv') + if (/^prebuild\/(centos|alpine)/.test(image)) argv.push('--tag-libc') + + return argv +} + +function cygwin (fp) { + if (process.platform !== 'win32') return fp + if (!truthy(process.env.COMPOSE_CONVERT_WINDOWS_PATHS)) return fp + + const unix = unixify(fp) + const drive = fp.match(/^([A-Z]):/i) + + return drive ? '/' + drive[1].toLowerCase() + unix : unix +} + +function truthy (str) { + return str === 'true' || str === '1' +} diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 98c4d90..0000000 --- a/package-lock.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "prebuildify-cross", - "version": "3.1.2", - "lockfileVersion": 1 -} diff --git a/package.json b/package.json index 99315e2..156689d 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,42 @@ { "name": "prebuildify-cross", "version": "3.1.2", - "description": "cross-compile prebuilds", - "bin": "./build", - "directories": { - "example": "examples" + "description": "Compile prebuilds in Docker", + "license": "GPL-3.0", + "bin": "./cli.js", + "scripts": { + "test": "standard" + }, + "dependencies": { + "browserify": "^16.5.0", + "docker-pull": "^1.1.0", + "docker-run": "^3.1.0", + "log-update": "^3.3.0", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "npm-packlist": "^2.0.1", + "once": "^1.4.0", + "pretty-bytes": "^5.3.0", + "tar-fs": "^2.0.0", + "unixify": "^1.0.0" + }, + "devDependencies": { + "standard": "^14.3.1" }, - "scripts": {}, "repository": { "type": "git", - "url": "git+ssh://git@github.com/ahdinosaur/prebuildify-cross.git" + "url": "https://github.com/prebuild/prebuildify-cross.git" }, - "keywords": [], - "author": "ahdinosaur", - "license": "GPL-3.0", "bugs": { - "url": "https://github.com/ahdinosaur/prebuildify-cross/issues" + "url": "https://github.com/prebuild/prebuildify-cross/issues" }, - "homepage": "https://github.com/ahdinosaur/prebuildify-cross#readme" + "homepage": "https://github.com/prebuild/prebuildify-cross", + "keywords": [ + "prebuildify", + "prebuild", + "dockcross" + ], + "engines": { + "node": ">=8" + } }