Factor out reproducibility flags for tar and gzip

WORKSPACE

@@ -220,9 +220,6 @@ nixpkgs_package(
     fail_not_supported = False,
     nix_file = "//nix:bazel.nix",
     nix_file_deps = common_nix_file_deps,
-    # Remove once we upgrade to Bazel >=3.0. Until then `nix-build` output
-    # confuses the JAR query in `daml-sdk-head`.
-    quiet = True,
     repositories = dev_env_nix_repos,
 )
 
@@ -243,9 +240,6 @@ nixpkgs_package(
     fail_not_supported = False,
     nix_file = "//nix:bazel.nix",
     nix_file_deps = common_nix_file_deps,
-    # Remove once we upgrade to Bazel >=3.0. Until then `nix-build` output
-    # confuses the JAR query in `daml-sdk-head`.
-    quiet = True,
     repositories = dev_env_nix_repos,
 )
 

bazel_tools/packaging/BUILD.bazel

@@ -1,7 +1,16 @@
 # Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-exports_files([
-    "packaging.bzl",
-    "package-app.sh",
-])
+load("@os_info//:os_info.bzl", "is_windows")
+
+sh_binary(
+    name = "package-app",
+    srcs = ["package-app.sh"],
+    data = [
+        "@gzip_dev_env//:gzip",
+        "@tar_dev_env//:tar",
+        "//bazel_tools/sh:mktgz",
+    ] + (["@patchelf_nix//:bin/patchelf"] if not is_windows else []),
+    visibility = ["//visibility:public"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

bazel_tools/packaging/package-app.sh

@@ -16,17 +16,61 @@
 # On Windows we only handle statically linked binaries
 # (Haskell binaries are linked statically on Windows) so we
 # just copy the binary and the resources and create a tarball from that.
+
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+# Make sure that runfiles and tools are still found after we change directory.
+case "$(uname -s)" in
+  Darwin)
+    abspath() { python -c 'import os.path, sys; sys.stdout.write(os.path.abspath(sys.argv[1]))' "$@"; }
+    canonicalpath() { python -c 'import os.path, sys; sys.stdout.write(os.path.realpath(sys.argv[1]))' "$@"; }
+    ;;
+  *)
+    abspath() { realpath -s "$@"; }
+    canonicalpath() { readlink -f "$@"; }
+    ;;
+esac
+
+if [[ -n ${RUNFILES_DIR:-} ]]; then
+  export RUNFILES_DIR=$(abspath $RUNFILES_DIR)
+fi
+if [[ -n ${RUNFILES_MANIFEST_FILE:-} ]]; then
+  export RUNFILES_DIR=$(abspath $RUNFILES_MANIFEST_FILE)
+fi
+
+case "$(uname -s)" in
+  Darwin|Linux)
+    tar=$(abspath $(rlocation tar_dev_env/tar))
+    gzip=$(abspath $(rlocation gzip_dev_env/gzip))
+    mktgz=$(abspath $(rlocation com_github_digital_asset_daml/bazel_tools/sh/mktgz))
+    patchelf=$(abspath $(rlocation patchelf_nix/bin/patchelf))
+    ;;
+  CYGWIN*|MINGW*|MSYS*)
+    tar=$(abspath $(rlocation tar_dev_env/usr/bin/tar.exe))
+    gzip=$(abspath $(rlocation gzip_dev_env/usr/bin/gzip.exe))
+    mktgz=$(abspath $(rlocation com_github_digital_asset_daml/bazel_tools/sh/mktgz.exe))
+    ;;
+esac
+
 set -eou pipefail
 
 WORKDIR="$(mktemp -d)"
 trap "rm -rf $WORKDIR" EXIT
 
-SRC=$1
-OUT=$2
+SRC=$(abspath $1)
+OUT=$(abspath $2)
 shift 2
 NAME=$(basename $SRC)
 mkdir -p $WORKDIR/$NAME/lib
-export ORIGIN=$(dirname $(readlink -f $SRC)) # for rpaths relative to binary
+export ORIGIN=$(dirname $(canonicalpath $SRC)) # for rpaths relative to binary
 
 # Copy in resources, if any.
 if [ $# -gt 0 ]; then
@@ -35,7 +79,7 @@ if [ $# -gt 0 ]; then
     if [[ "$res" == *.tar.gz ]]; then
       # If a resource is a tarball, e.g., because it originates from another
       # rule we extract it.
-      tar xf "$res" --strip-components=1 -C "$WORKDIR/$NAME/resources"
+      $tar xf "$res" --strip-components=1 -C "$WORKDIR/$NAME/resources"
     else
       cp -aL "$res" "$WORKDIR/$NAME/resources"
     fi
@@ -47,13 +91,13 @@ if [ "$(uname -s)" == "Linux" ]; then
   binary="$WORKDIR/$NAME/lib/$NAME"
   cp $SRC $binary
   chmod u+w $binary
-  rpaths_binary=$(patchelf --print-rpath "$binary"|tr ':' ' ')
+  rpaths_binary=$($patchelf --print-rpath "$binary"|tr ':' ' ')
   function copy_deps {
     local from target needed libOK rpaths
     from=$1
     target=$2
-    needed=$(patchelf --print-needed "$from")
-    rpaths="$(patchelf --print-rpath "$from"|tr ':' ' ') $rpaths_binary"
+    needed=$($patchelf --print-needed "$from")
+    rpaths="$($patchelf --print-rpath "$from"|tr ':' ' ') $rpaths_binary"
 
     for lib in $needed; do
       if [ ! -f "$target/$lib" ]; then
@@ -67,7 +111,7 @@ if [ "$(uname -s)" == "Linux" ]; then
             if [ "$lib" != "ld-linux-x86-64.so.2" ]; then
               # clear the old rpaths (silence stderr as it always warns
               # with "working around a Linux kernel bug".
-              patchelf --set-rpath '$ORIGIN' "$target/$lib" 2> /dev/null
+              $patchelf --set-rpath '$ORIGIN' "$target/$lib" 2> /dev/null
             fi
             copy_deps "$rpath/$lib" "$target"
             break
@@ -94,8 +138,8 @@ if [ "$(uname -s)" == "Linux" ]; then
    done
   )
 
-  patchelf --set-rpath '$ORIGIN/lib' "$binary"
-  patchelf --set-interpreter ld-undefined.so "$binary"
+  $patchelf --set-rpath '$ORIGIN/lib' "$binary"
+  $patchelf --set-interpreter ld-undefined.so "$binary"
 
   # Link resources directory to lib, as that'll be the actual
   # binary's location.
@@ -116,7 +160,7 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then
   cp $SRC $WORKDIR/$NAME/$NAME
   chmod u+w $WORKDIR/$NAME/$NAME
   function copy_deps() {
-    local from_original=$(readlink -f $1)
+    local from_original=$(canonicalpath $1)
     local from_copied=$2
     local needed="$(/usr/bin/otool -L "$from_copied" | sed -n -e '1d' -e 's/^\s*\([^ ]*\).*$/\1/p')"
     # Note that it is crucial that we resolve loader_path relative to from_original instead of from_copied
@@ -173,6 +217,4 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then
 else
     cp "$SRC" "$WORKDIR/$NAME/$NAME"
 fi
-cd $WORKDIR && tar c $NAME \
-    --owner=0 --group=0 --numeric-owner --mtime=2000-01-01\ 00:00Z --sort=name \
-    | gzip -n > $OUT
+cd $WORKDIR && $mktgz $OUT $NAME

bazel_tools/packaging/packaging.bzl

@@ -6,37 +6,19 @@
 load("@os_info//:os_info.bzl", "is_windows")
 
 def _package_app_impl(ctx):
-    files = depset(ctx.attr.binary.files)
-    runfiles = ctx.attr.binary.default_runfiles.files
-    datafiles = ctx.attr.binary[DefaultInfo].data_runfiles.files
-
     args = ctx.actions.args()
-    inputs = depset([], transitive = [files, runfiles, datafiles] + [r.files for r in ctx.attr.resources])
-    tools = [ctx.executable.tar, ctx.executable.gzip] if is_windows else [ctx.executable.patchelf, ctx.executable.tar, ctx.executable.gzip]
-    ctx.actions.run_shell(
+    args.add(ctx.executable.binary.path)
+    args.add(ctx.outputs.out.path)
+    args.add_all(ctx.attr.resources, map_each = _get_resource_path)
+    ctx.actions.run(
+        executable = ctx.executable.package_app,
         outputs = [ctx.outputs.out],
-        tools = [ctx.executable.package_app] + tools,
-        inputs = inputs.to_list(),
+        inputs = ctx.files.resources,
+        # Binaries are passed through tools so that Bazel can make the runfiles
+        # tree available to the action.
+        tools = [ctx.executable.binary],
         arguments = [args],
         progress_message = "Packaging " + ctx.attr.name,
-        command = """
-      set -eu
-      export PATH=$PATH:{path}
-      {package_app} \
-        "$PWD/{binary}" \
-        "$PWD/{output}" \
-        {resources}
-    """.format(
-            path = ":".join(["$PWD/`dirname {tool}`".format(tool = tool.path) for tool in tools]),
-            output = ctx.outputs.out.path,
-            name = ctx.attr.name,
-            package_app = ctx.executable.package_app.path,
-            binary = ctx.executable.binary.path,
-            resources = " ".join([
-                _get_resource_path(r)
-                for r in ctx.attr.resources
-            ]),
-        ),
     )
 
 def _get_resource_path(r):
@@ -71,26 +53,8 @@ package_app = rule(
         "resources": attr.label_list(
             allow_files = True,
         ),
-        "patchelf": attr.label(
-            default = None if is_windows else Label("@patchelf_nix//:bin/patchelf"),
-            cfg = "host",
-            executable = True,
-            allow_files = True,
-        ),
-        "tar": attr.label(
-            default = Label("@tar_dev_env//:tar"),
-            cfg = "host",
-            executable = True,
-            allow_files = True,
-        ),
-        "gzip": attr.label(
-            default = Label("@gzip_dev_env//:gzip"),
-            cfg = "host",
-            executable = True,
-            allow_files = True,
-        ),
         "package_app": attr.label(
-            default = Label("//bazel_tools/packaging:package-app.sh"),
+            default = Label("//bazel_tools/packaging:package-app"),
             cfg = "host",
             executable = True,
             allow_files = True,

bazel_tools/sh/BUILD.bazel

@@ -2,3 +2,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 exports_files(["test.sh.tpl"])
+
+sh_binary(
+    name = "mktar",
+    srcs = ["mktar.sh"],
+    data = ["@tar_dev_env//:tar"],
+    visibility = ["//visibility:public"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
+sh_binary(
+    name = "mktgz",
+    srcs = ["mktgz.sh"],
+    data = [
+        "@gzip_dev_env//:gzip",
+        "@tar_dev_env//:tar",
+    ],
+    visibility = ["//visibility:public"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

bazel_tools/sh/mktar.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+set -euo pipefail
+usage() {
+  cat >&2 <<'EOF'
+usage: mktar OUTPUT ARGS...
+
+Creates an uncompressed tarball in OUTPUT passing ARGS to tar. The created
+tarball is reproducible, i.e. it does not contain any timestamps or similar
+non-deterministic inputs. See https://reproducible-builds.org/docs/archives/
+EOF
+}
+trap usage ERR
+
+case "$(uname -s)" in
+  Darwin|Linux)
+    tar=$(rlocation tar_dev_env/tar)
+    ;;
+  CYGWIN*|MINGW*|MSYS*)
+    tar=$(rlocation tar_dev_env/usr/bin/tar.exe)
+    ;;
+esac
+
+$tar cf "$1" "${@:2}" \
+  --owner="0" \
+  --group="0" \
+  --numeric-owner \
+  --mtime="2000-01-01 00:00Z" \
+  --no-acls \
+  --no-xattrs \
+  --no-selinux \
+  --sort="name" \
+  --format=ustar

bazel_tools/sh/mktgz.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Copy-pasted from the Bazel Bash runfiles library v2.
+set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v2 ---
+
+set -euo pipefail
+usage() {
+  cat >&2 <<'EOF'
+usage: mktgz OUTPUT ARGS...
+
+Creates a gzip compressed tarball in OUTPUT passing ARGS to tar. The created
+tarball is reproducible, i.e. it does not contain any timestamps or similar
+non-deterministic inputs. See https://reproducible-builds.org/docs/archives/
+EOF
+}
+trap usage ERR
+
+case "$(uname -s)" in
+  Darwin|Linux)
+    tar=$(rlocation tar_dev_env/tar)
+    gzip=$(rlocation gzip_dev_env/gzip)
+    ;;
+  CYGWIN*|MINGW*|MSYS*)
+    tar=$(rlocation tar_dev_env/usr/bin/tar.exe)
+    gzip=$(rlocation gzip_dev_env/usr/bin/gzip.exe)
+    ;;
+esac
+
+$tar c "${@:2}" \
+  --owner="0" \
+  --group="0" \
+  --numeric-owner \
+  --mtime="2000-01-01 00:00Z" \
+  --no-acls \
+  --no-xattrs \
+  --no-selinux \
+  --sort="name" \
+  --format=ustar \
+  | $gzip -n > "$1"