Split AppImage mnemonic into MkAppDir, Sqfstar, AppImage and …

…set `resource_set` for mksquashfs

appimage/appimage.bzl

@@ -3,60 +3,105 @@
 load("@rules_appimage//appimage/private:mkapprun.bzl", "make_apprun")
 load("@rules_appimage//appimage/private:runfiles.bzl", "collect_runfiles_info")
 
+MKSQUASHFS_ARGS = [
+    "-exit-on-error",
+    "-no-progress",
+    "-quiet",
+    "-all-root",
+    "-reproducible",
+    "-mkfs-time",
+    "0",
+    "-root-time",
+    "0",
+]
+MKSQUASHFS_NUM_PROCS = 8
+MKSQUASHFS_MEM_MB = 4096
+
+def _resources(*_args, **_kwargs):
+    """See https://bazel.build/rules/lib/builtins/actions#run.resource_set."""
+    return {"cpu": MKSQUASHFS_NUM_PROCS, "memory": MKSQUASHFS_MEM_MB}
+
 def _appimage_impl(ctx):
     """Implementation of the appimage rule."""
     toolchain = ctx.toolchains["//appimage:appimage_toolchain_type"]
 
-    tools = depset(
-        direct = [ctx.executable._tool],
-        transitive = [ctx.attr._tool[DefaultInfo].default_runfiles.files],
-    )
     runfile_info = collect_runfiles_info(ctx)
-    manifest_file = ctx.actions.declare_file(ctx.attr.name + "-manifest.json")
+    manifest_file = ctx.actions.declare_file(ctx.attr.name + ".manifest.json")
     ctx.actions.write(manifest_file, json.encode_indent(runfile_info.manifest))
     apprun = make_apprun(ctx)
-    inputs = depset(direct = [manifest_file, apprun, toolchain.appimage_runtime] + runfile_info.files)
+    inputs = depset(direct = [manifest_file, apprun] + runfile_info.files)
 
+    # Create the AppDir tar
+    appdirtar = ctx.actions.declare_file(ctx.attr.name + ".tar")
     args = ctx.actions.args()
-    args.add_all(["--manifest", manifest_file.path])
-    args.add_all(["--apprun", apprun.path])
-    args.add_all(["--runtime", toolchain.appimage_runtime.path])
-    args.add_all(ctx.attr.build_args, format_each = "--mksquashfs_arg=%s")
-    args.add(ctx.outputs.executable.path)
+    args.add("--manifest").add(manifest_file.path)
+    args.add("--apprun").add(apprun.path)
+    args.add(appdirtar.path)
+    ctx.actions.run(
+        mnemonic = "MkAppDir",
+        inputs = inputs,
+        executable = ctx.executable._mkappdir,
+        arguments = [args],
+        outputs = [appdirtar],
+    )
+
+    # Create the AppDir squashfs
+    args = ctx.actions.args()
+    args.add_all(MKSQUASHFS_ARGS)
+    args.add("-processors").add(MKSQUASHFS_NUM_PROCS)
+    args.add("-mem").add("{}M".format(MKSQUASHFS_MEM_MB))
+    mksquashfs_args = MKSQUASHFS_ARGS + ctx.attr.build_args + ["-tar"]
+    appdirsqfs = ctx.actions.declare_file(ctx.attr.name + ".sqfs")
+    ctx.actions.run_shell(
+        mnemonic = "Sqfstar",
+        inputs = [appdirtar],
+        command = "{exe} - {dst} {args} <{src}".format(
+            exe = ctx.executable._mksquashfs.path,
+            dst = appdirsqfs.path,
+            args = " ".join(mksquashfs_args),
+            src = appdirtar.path,
+        ),
+        tools = [ctx.executable._mksquashfs],
+        outputs = [appdirsqfs],
+        resource_set = _resources,
+    )
+
+    # Create the Appimage
+    ctx.actions.run_shell(
+        mnemonic = "AppImage",
+        inputs = [toolchain.appimage_runtime, appdirsqfs],
+        command = "cat $1 $2 >$3",
+        arguments = [
+            toolchain.appimage_runtime.path,
+            appdirsqfs.path,
+            ctx.outputs.executable.path,
+        ],
+        outputs = [ctx.outputs.executable],
+    )
 
     # Take the `binary` env and add the appimage target's env on top of it
     env = {}
     if RunEnvironmentInfo in ctx.attr.binary:
         env.update(ctx.attr.binary[RunEnvironmentInfo].environment)
     env.update(ctx.attr.env)
 
-    # Run our tool to create the AppImage
-    ctx.actions.run(
-        mnemonic = "AppImage",
-        inputs = inputs,
-        env = ctx.attr.build_env,
-        executable = ctx.executable._tool,
-        arguments = [args],
-        outputs = [ctx.outputs.executable],
-        tools = tools,
-    )
-
     return [
         DefaultInfo(
             executable = ctx.outputs.executable,
             files = depset([ctx.outputs.executable]),
             runfiles = ctx.runfiles(files = [ctx.outputs.executable]),
         ),
         RunEnvironmentInfo(env),
+        OutputGroupInfo(appimage_debug = depset([manifest_file, appdirtar, appdirsqfs])),
     ]
 
 _ATTRS = {
     "binary": attr.label(executable = True, cfg = "target"),
     "build_args": attr.string_list(),
-    "build_env": attr.string_dict(),
     "data": attr.label_list(allow_files = True, doc = "Any additional data that will be made available inside the appimage"),
     "env": attr.string_dict(doc = "Runtime environment variables. See https://bazel.build/reference/be/common-definitions#common-attributes-tests"),
-    "_tool": attr.label(default = "//appimage/private/tool", executable = True, cfg = "exec"),
+    "_mkappdir": attr.label(default = "//appimage/private:mkappdir", executable = True, cfg = "exec"),
+    "_mksquashfs": attr.label(default = "@squashfs-tools//:mksquashfs", executable = True, cfg = "exec"),
 }
 
 appimage = rule(

appimage/private/BUILD

@@ -0,0 +1,7 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+py_binary(
+    name = "mkappdir",
+    srcs = ["mkappdir.py"],
+    visibility = ["//visibility:public"],
+)

appimage/private/tool/mkappimage.py → appimage/private/mkappdir.py

@@ -1,49 +1,25 @@
-"""Library to prepare and build AppImages."""
+"""Prepare and build an AppImage AppDir tarball."""
 
 from __future__ import annotations
 
+import argparse
 import copy
 import functools
 import json
 import os
 import shutil
-import subprocess
+import sys
+import tarfile
 import tempfile
 from pathlib import Path
-from typing import TYPE_CHECKING, NamedTuple
-
-from python import runfiles as bazel_runfiles
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Sequence
 
     _ManifestDataT = dict[str, list[str | dict[str, str]]]
 
 
-def _get_path_or_raise(path: str) -> Path:
-    """Return a Path to a file in the runfiles, or raise FileNotFoundError."""
-    runfiles = bazel_runfiles.Create()
-    if not runfiles:
-        raise FileNotFoundError("Could not find runfiles")
-    runfile = runfiles.Rlocation(path)
-    if not runfile:
-        raise FileNotFoundError(f"Could not find {path} in runfiles")
-    if not Path(runfile).exists():
-        raise FileNotFoundError(f"{runfile} does not exist")
-    return Path(runfile)
-
-
-MKSQUASHFS = _get_path_or_raise("squashfs-tools/mksquashfs")
-
-
-class AppDirParams(NamedTuple):
-    """Parameters for the AppDir."""
-
-    manifest: Path
-    apprun: Path
-    runtime: Path
-
-
 def relative_path(target: Path, origin: Path) -> Path:
     """Return path of target relative to origin."""
     try:
@@ -166,7 +142,7 @@ def _move_relative_symlinks_in_files_to_their_own_section(manifest_data: _Manife
     return new_manifest_data
 
 
-def populate_appdir(appdir: Path, params: AppDirParams) -> None:
+def populate_appdir(appdir: Path, manifest: Path) -> None:
     """Make the AppDir that will be squashfs'd into the AppImage.
 
     Note that the [AppImage Type2 Spec][appimage-spec] specifies that the contained [AppDir][appdir-spec] may contain a
@@ -178,7 +154,7 @@ def populate_appdir(appdir: Path, params: AppDirParams) -> None:
     [appimaged]: https://docs.appimage.org/user-guide/run-appimages.html#integrating-appimages-into-the-desktop
     """
     appdir.mkdir(parents=True, exist_ok=True)
-    manifest_data = json.loads(params.manifest.read_text())
+    manifest_data = json.loads(manifest.read_text())
     manifest_data = _move_relative_symlinks_in_files_to_their_own_section(manifest_data)
 
     for empty_file in manifest_data["empty_files"]:
@@ -234,25 +210,42 @@ def populate_appdir(appdir: Path, params: AppDirParams) -> None:
         assert src.exists(), f"want to copy {src} to {dst}, but it does not exist"
         _copy_file_or_dir(src, dst, keep_symlinks=False)
 
-    apprun_path = appdir / "AppRun"
-    shutil.copy2(params.apprun, apprun_path)
-    apprun_path.chmod(0o751)
-
-
-def make_squashfs(params: AppDirParams, mksquashfs_params: Iterable[str], output_path: str) -> None:
-    """Run mksquashfs to create the squashfs filesystem for the appimage."""
-    with tempfile.TemporaryDirectory(suffix="AppDir") as tmpdir_name:
-        populate_appdir(appdir=Path(tmpdir_name), params=params)
-        cmd = [os.fspath(MKSQUASHFS), tmpdir_name, output_path, "-root-owned", "-noappend", *mksquashfs_params]
-        try:
-            subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError as exc:
-            raise RuntimeError(f"Failed to run {' '.join(cmd)!r} (returned {exc.returncode}): {exc.stdout}") from exc
-
 
-def make_appimage(params: AppDirParams, mksquashfs_params: Iterable[str], output_path: Path) -> None:
-    """Make the AppImage by concatenating the AppImage runtime and the AppDir squashfs."""
-    shutil.copy2(src=params.runtime, dst=output_path)
-    with output_path.open(mode="ab") as output_file, tempfile.NamedTemporaryFile(mode="w+b") as tmp_squashfs:
-        make_squashfs(params, mksquashfs_params, tmp_squashfs.name)
-        shutil.copyfileobj(tmp_squashfs, output_file)
+def _make_executable(ti: tarfile.TarInfo) -> tarfile.TarInfo:
+    ti.mode |= 0o111
+    return ti
+
+
+def make_appdir_tar(manifest: Path, apprun: Path, output: Path) -> None:
+    """Create an AppImage AppDir (uncompressed) tar ready to be sqfstar'ed."""
+    with tarfile.open(output, "w") as tar:
+        tar.add(apprun.resolve(), arcname="AppRun", filter=_make_executable)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            appdir = Path(tmpdir)
+            populate_appdir(appdir, manifest)
+            for top_level in appdir.iterdir():
+                tar.add(top_level, arcname=top_level.name)
+
+
+def parse_args(args: Sequence[str]) -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(description="Prepare and build AppImages.")
+    parser.add_argument(
+        "--manifest",
+        required=True,
+        type=Path,
+        help="Path to manifest json with file and link definitions, e.g. 'bazel-bin/tests/appimage_py-manifest.json'",
+    )
+    parser.add_argument(
+        "--apprun",
+        required=True,
+        type=Path,
+        help="Path to AppRun script",
+    )
+    parser.add_argument("output", type=Path, help="Where to place output AppDir tar")
+    return parser.parse_args(args)
+
+
+if __name__ == "__main__":
+    args = parse_args(sys.argv[1:])
+    make_appdir_tar(args.manifest, args.apprun, args.output)

tests/BUILD

@@ -139,10 +139,6 @@ py_test(
 appimage(
     name = "external_bin.appimage",
     binary = "@rules_python//tools:wheelmaker",
-    build_env = {
-        # Example: Set environment variable for mksquashfs to set all file timestamps to 1970-01-01.
-        "SOURCE_DATE_EPOCH": "0",
-    },
 )
 
 sh_test(