From 85cf20c1d8c19e1b6072ac663123a5f15cb5815a Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Tue, 27 Aug 2019 16:52:17 -0400 Subject: [PATCH 1/8] bin/nvh: xz compression (w/ environment variable) Set "NVH_USE_XZ" to "true" to use xz compression. --- bin/nvh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/nvh b/bin/nvh index b6a36e0..6f0ef6d 100755 --- a/bin/nvh +++ b/bin/nvh @@ -568,7 +568,11 @@ function display_compatible_file_field { function tarball_url() { local version="$1" - echo "${g_mirror_url}/${version}/node-${version}-$(display_tarball_platform).tar.gz" + + local ext="gz" + if [[ "${NVH_USE_XZ}" = "true" ]]; then local ext="xz"; fi + + echo "${g_mirror_url}/${version}/node-${version}-$(display_tarball_platform).tar.${ext}" } # @@ -664,6 +668,9 @@ function install() { local url="$(tarball_url "${resolved_version}")" is_ok "${url}" || abort "download preflight failed for '$resolved_version' (${url})" + local tarflags="-zx" + if [[ "${NVH_USE_XZ}" = "true" ]]; then local tarflags="-Jx"; fi + install_log "mkdir" "${dir}" mkdir -p "${dir}" || abort "sudo required (or change ownership, or define NVH_PREFIX)" touch "${dir}/nvh.lock" @@ -671,7 +678,7 @@ function install() { cd "${dir}" || abort "Failed to cd to directory" install_log "fetch" "${url}" - do_get "${url}" | tar -zx --strip-components=1 + do_get "${url}" | tar "${tarflags}" --strip-components=1 [[ "${GET_OPTIONS_SHOW_PROGRESS}" = "true" ]] && erase_line rm -f "${dir}/nvh.lock" From 890ae1b5918bfc16995989757388bbd3eb776779 Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Tue, 27 Aug 2019 17:12:02 -0400 Subject: [PATCH 2/8] bin/nvh: Command-line argument for xz compression --use-xz for xz, --no-use-xz for gzip. Overrides environment variable. --- bin/nvh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/nvh b/bin/nvh index 6f0ef6d..a7614fa 100755 --- a/bin/nvh +++ b/bin/nvh @@ -294,6 +294,8 @@ Options: -h, --help Display help information --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) -p, --preserve Preserve npm and npx during install of node + --use-xz Download xz-compressed tarballs, if available and extractable by tar, during install of node (this is the default) + --no-use-xz Download gzip-compressed tarballs, rather than xz-compressed ones, during install of node -V, --version Output version of nvh Aliases: @@ -1159,6 +1161,8 @@ function main() { --insecure) set_insecure ;; -p|--preserve) NVH_PRESERVE_NPM="true" ;; --no-preserve) NVH_PRESERVE_NPM="" ;; + --use-xz) NVH_USE_XZ="true" ;; + --no-use-xz) NVH_USE_XZ="false" ;; -V|--version) echo "${VERSION}"; exit ;; -*) abort "unrecognised option '$1'"; exit 1 ;; exec) unprocessed_args=("$@"); break ;; From 7418730beaa2cc8572797e72adcfaa043a6b6a91 Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Sat, 21 Sep 2019 15:24:14 -0400 Subject: [PATCH 3/8] bin/nvh: Check node version >= 4 for xz Node.js has had xz support on Linux and macOS since 4.0.0 (io.js had xz starting from 1.0+ for Linux, from 2.3.2+ for macOS. Node.js picks up where io.js left off, so 4.0+ has xz for both Linux and macOS as well.) There are some versions of Node.js in the 0.10 and 0.12 series that also have xz tarballs, but these aren't downloaded often, and checking just the major version is simpler than checking major, minor and patch versions. --- bin/nvh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/nvh b/bin/nvh index a7614fa..d6a24f6 100755 --- a/bin/nvh +++ b/bin/nvh @@ -667,6 +667,14 @@ function install() { echo install_log "installing" "${g_mirror_folder_name}/${resolved_version}" + if [[ "${NVH_USE_XZ}" = "true" ]]; then + local resolved_major_version + resolved_major_version="$(echo ${resolved_version#v} | cut -d '.' -f '1')" + if [[ "${resolved_major_version}" -lt 4 ]]; then + NVH_USE_XZ="false" + fi + fi + local url="$(tarball_url "${resolved_version}")" is_ok "${url}" || abort "download preflight failed for '$resolved_version' (${url})" From 8ec7b715b0021288e2dc95df3fa20926db8b633a Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Sat, 31 Aug 2019 10:01:21 -0400 Subject: [PATCH 4/8] bin/nvh: Add "can_use_xz()" function Determine whether xz can be used, based on platform/system details. --- bin/nvh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bin/nvh b/bin/nvh index d6a24f6..564cc2d 100755 --- a/bin/nvh +++ b/bin/nvh @@ -517,6 +517,18 @@ function is_ok() { fi } +# +# Synopsis: can_use_xz +# + +function can_use_xz() { + # nvh uses xz on Linux and macOS at this time, if the system supports it + if ! ([[ "$(uname -s)" = "Linux" ]] && command -v xz &> /dev/null) \ + && ! ([[ "$(uname -s)" = "Darwin" ]] && [[ "$(sw_vers -productVersion | cut -d '.' -f 2)" -gt "8" ]]); then + return 1 + fi +} + # # Synopsis: display_tarball_platform # From 0c17df1a5d18968842eff32ffaaa127f427072aa Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Sat, 21 Sep 2019 13:11:44 -0400 Subject: [PATCH 5/8] bin/nvh: Use can_use_xz() to gate xz usage If can_use_xz() checks fail, revert to using gzip. --- bin/nvh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/nvh b/bin/nvh index 564cc2d..f643cbb 100755 --- a/bin/nvh +++ b/bin/nvh @@ -676,6 +676,8 @@ function install() { fi fi + [[ "${NVH_USE_XZ}" = "true" ]] && can_use_xz || NVH_USE_XZ="false" + echo install_log "installing" "${g_mirror_folder_name}/${resolved_version}" From 40883c1b5c1de93f4dd8786bad6399ab00220773 Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Wed, 28 Aug 2019 15:22:32 -0400 Subject: [PATCH 6/8] bin/nvh: Normalize "NVH_USE_XZ" from environment Adapted from this comment: https://github.com/shadowspawn/nvh/issues/8#issuecomment-533034644 Co-authored-by: John Gee --- bin/nvh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/nvh b/bin/nvh index f643cbb..033f88f 100755 --- a/bin/nvh +++ b/bin/nvh @@ -22,6 +22,16 @@ NVH_NODE_DOWNLOAD_MIRROR=${NVH_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download NVH_NODE_DOWNLOAD_MIRROR=${NVH_NODE_DOWNLOAD_MIRROR%/} readonly NVH_NODE_DOWNLOAD_MIRROR +# Enabled by default. User may set NVH_USE_XZ to 0 to disable. +# Normalise to true/false. +# May be overridden by command line flags. + +if [[ "${NVH_USE_XZ}" = "0" ]]; then + NVH_USE_XZ="false" +else + NVH_USE_XZ="true" +fi + # NVH_PRESERVE_NPM tested with -z -n # made readonly after CLI option parsing From 0bea6c990d048a4567c9b39e328fabe99c84bb71 Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Sat, 21 Sep 2019 13:37:59 -0400 Subject: [PATCH 7/8] bin/nvh: Add "forced on" state for xz Skips node version checks, which would otherwise gate xz usage. In other words: Overrides any checks that would normally be run, so that nvh always attempts to use xz. --- Also updates "--use-xz" to not set the "xz forced on" state. (This matches the flag's description in the help messages.) --- Partly adapted from this comment (with liberties taken on implementation): https://github.com/shadowspawn/nvh/issues/8#issuecomment-533034644 Co-authored-by: John Gee --- bin/nvh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/nvh b/bin/nvh index 033f88f..98db794 100755 --- a/bin/nvh +++ b/bin/nvh @@ -22,12 +22,16 @@ NVH_NODE_DOWNLOAD_MIRROR=${NVH_NODE_DOWNLOAD_MIRROR:-https://nodejs.org/download NVH_NODE_DOWNLOAD_MIRROR=${NVH_NODE_DOWNLOAD_MIRROR%/} readonly NVH_NODE_DOWNLOAD_MIRROR -# Enabled by default. User may set NVH_USE_XZ to 0 to disable. -# Normalise to true/false. +# Using xz instead of gzip is enabled by default, if xz compatibility checks pass. +# User may set NVH_USE_XZ to 0 to disable, or set to anything else to force-enable. # May be overridden by command line flags. +g_xz_force_enabled="false" if [[ "${NVH_USE_XZ}" = "0" ]]; then NVH_USE_XZ="false" +elif [[ -n "${NVH_USE_XZ+defined}" ]]; then + NVH_USE_XZ="true" + g_xz_force_enabled="true" else NVH_USE_XZ="true" fi @@ -686,12 +690,14 @@ function install() { fi fi - [[ "${NVH_USE_XZ}" = "true" ]] && can_use_xz || NVH_USE_XZ="false" + if [[ "${NVH_USE_XZ}" = true ]] && [[ "S{g_xz_force_enabled}" = "false" ]]; then + can_use_xz || NVH_USE_XZ="false" + fi echo install_log "installing" "${g_mirror_folder_name}/${resolved_version}" - if [[ "${NVH_USE_XZ}" = "true" ]]; then + if [[ "${NVH_USE_XZ}" = "true" ]] && [[ "${g_xz_force_enabled}" = "false" ]]; then local resolved_major_version resolved_major_version="$(echo ${resolved_version#v} | cut -d '.' -f '1')" if [[ "${resolved_major_version}" -lt 4 ]]; then @@ -1193,7 +1199,7 @@ function main() { --insecure) set_insecure ;; -p|--preserve) NVH_PRESERVE_NPM="true" ;; --no-preserve) NVH_PRESERVE_NPM="" ;; - --use-xz) NVH_USE_XZ="true" ;; + --use-xz) NVH_USE_XZ="true"; g_xz_force_enabled="false" ;; --no-use-xz) NVH_USE_XZ="false" ;; -V|--version) echo "${VERSION}"; exit ;; -*) abort "unrecognised option '$1'"; exit 1 ;; From ea52a42dd2e2773270f3292548e295ad3ae3f24a Mon Sep 17 00:00:00 2001 From: DeeDeeG Date: Sat, 21 Sep 2019 13:55:59 -0400 Subject: [PATCH 8/8] bin/nvh: Add --force-use-xz command-line flag Activates "force use xz" state, which skips xz-related sanity checks. --- bin/nvh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/nvh b/bin/nvh index 98db794..4b791e9 100755 --- a/bin/nvh +++ b/bin/nvh @@ -309,6 +309,7 @@ Options: --insecure Turn off certificate checking for https requests (may be needed from behind a proxy server) -p, --preserve Preserve npm and npx during install of node --use-xz Download xz-compressed tarballs, if available and extractable by tar, during install of node (this is the default) + --force-use-xz Download xz-compressed tarballs. May fail if xz tarball is not available and extractable --no-use-xz Download gzip-compressed tarballs, rather than xz-compressed ones, during install of node -V, --version Output version of nvh @@ -1200,6 +1201,7 @@ function main() { -p|--preserve) NVH_PRESERVE_NPM="true" ;; --no-preserve) NVH_PRESERVE_NPM="" ;; --use-xz) NVH_USE_XZ="true"; g_xz_force_enabled="false" ;; + --force-use-xz) NVH_USE_XZ="true"; g_xz_force_enabled="true" ;; --no-use-xz) NVH_USE_XZ="false" ;; -V|--version) echo "${VERSION}"; exit ;; -*) abort "unrecognised option '$1'"; exit 1 ;;