clairctl: uniform import/export compression

Clairctl's import and export functionality got out-of-sync such that
"import" couldn't handle "export" output unchanged. This change brings
them in-line and has them behave similarly, with similar flags.

Closes: PROJQUAY-2367
Signed-off-by: Hank Donnay <hdonnay@redhat.com>

Documentation/concepts/updatersandairgap.md

@@ -70,17 +70,17 @@ For example:
 
 ```sh
 # On a workstation, run:
-clairctl export-updaters updates.gz
+clairctl export-updaters updates.json.gz
 ```
 
 ```sh
 # Move the resulting file to a place reachable by the cluster:
-scp updates.gz internal-webserver:/var/www/
+scp updates.json.gz internal-webserver:/var/www/
 ```
 
 ```sh
 # On a pod inside the cluster, import the file:
-clairctl import-updaters http://web.svc/updates.gz
+clairctl import-updaters http://web.svc/updates.json.gz
 ```
 
 Note that a configuration file is needed to run these commands.

cmd/clairctl/export.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strings"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/quay/claircore/libvuln/driver"
 	"github.com/quay/claircore/libvuln/jsonblob"
 	"github.com/quay/claircore/libvuln/updates"
@@ -22,18 +25,32 @@ var ExportCmd = &cli.Command{
 	Name:      "export-updaters",
 	Action:    exportAction,
 	Usage:     "run updaters and export results",
-	ArgsUsage: "[out]",
+	ArgsUsage: "[out[.gz|.zst]]",
 	Flags: []cli.Flag{
 		// Strict can be used to check that updaters still work.
 		&cli.BoolFlag{
 			Name:  "strict",
 			Usage: "Return non-zero exit when updaters report errors.",
 		},
+		&cli.BoolFlag{
+			Name:    "gzip",
+			Aliases: []string{"g"},
+			Usage:   "Compress output with gzip.",
+		},
+		&cli.BoolFlag{
+			Name:    "zstd",
+			Aliases: []string{"z"},
+			Usage:   "Compress output with zstd.",
+		},
 	},
 	Description: `Run configured exporters and export to a file.
 
-   A configuration file is needed to run this command, see 'clairctl help'
-   for how to specify one.`, // NB this has spaces, not tabs.
+If a file name is supplied and ends with ".gz" or ".zst" and neither the
+"z" or "g" flag have been supplied, output will be compressed with gzip
+or zstd compression, respectively.
+
+A configuration file is needed to run this command, see 'clairctl help'
+for how to specify one.`,
 }
 
 func exportAction(c *cli.Context) error {
@@ -52,9 +69,38 @@ func exportAction(c *cli.Context) error {
 		}
 		defer f.Close()
 		out = f
+		switch {
+		case c.IsSet("zstd") || c.IsSet("gzip"):
+			break
+		case strings.HasSuffix(args.First(), ".zst"):
+			c.Set("zstd", "true")
+		case strings.HasSuffix(args.First(), ".gz"):
+			c.Set("gzip", "true")
+		}
 	default:
 		return errors.New("too many arguments (wanted at most one)")
 	}
+	switch {
+	case c.Bool("zstd"):
+		enc, err := zstd.NewWriter(out)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := enc.Close(); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+			}
+		}()
+		out = enc
+	case c.Bool("gzip"):
+		enc := gzip.NewWriter(out)
+		defer func() {
+			if err := enc.Close(); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+			}
+		}()
+		out = enc
+	}
 
 	// Read and process the config file.
 	cfg, err := loadConfig(c.String("config"))

cmd/clairctl/import.go

@@ -1,15 +1,18 @@
 package main
 
 import (
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/klauspost/compress/zstd"
 	"github.com/quay/claircore/libvuln"
 	"github.com/urfave/cli/v2"
 
@@ -21,12 +24,27 @@ var ImportCmd = &cli.Command{
 	Name:      "import-updaters",
 	Action:    importAction,
 	Usage:     "import updates",
-	ArgsUsage: "input...",
-	Flags:     []cli.Flag{},
-	Description: `Import updates from files or HTTP URIs.
+	ArgsUsage: "input[.gz|.zst]|-",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "gzip",
+			Aliases: []string{"g"},
+			Usage:   "Decompress input with gzip.",
+		},
+		&cli.BoolFlag{
+			Name:    "zstd",
+			Aliases: []string{"z"},
+			Usage:   "Decompress input with zstd.",
+		},
+	},
+	Description: `Import updates from a file or HTTP URI.
 
-   A configuration file is needed to run this command, see 'clairctl help'
-   for how to specify one.`, // NB this has spaces, not tabs.
+If the supplied file name ends with ".gz" or ".zst" and neither the "z"
+or "g" flag have been supplied, input will be decompressed with gzip or
+zstd compression, respectively.
+
+A configuration file is needed to run this command, see 'clairctl help'
+for how to specify one.`,
 }
 
 func importAction(c *cli.Context) error {
@@ -45,15 +63,43 @@ func importAction(c *cli.Context) error {
 	// Setup the input file.
 	args := c.Args()
 	if args.Len() != 1 {
-		return errors.New("need at least one argument")
+		return errors.New("need one argument")
 	}
 	inName := args.First()
+	switch {
+	case c.IsSet("zstd") || c.IsSet("gzip"):
+		break
+	case strings.HasSuffix(inName, ".zst"):
+		c.Set("zstd", "true")
+	case strings.HasSuffix(inName, ".gz"):
+		c.Set("gzip", "true")
+	}
 
 	in, err := openInput(ctx, cl, inName)
 	if err != nil {
 		return err
 	}
 	defer in.Close()
+	switch {
+	case c.Bool("zstd"):
+		dec, err := zstd.NewReader(in)
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		in = io.NopCloser(dec)
+	case c.Bool("gzip"):
+		dec, err := gzip.NewReader(in)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := dec.Close(); err != nil {
+				fmt.Fprintln(os.Stderr, err)
+			}
+		}()
+		in = dec
+	}
 
 	pool, err := pgxpool.Connect(ctx, cfg.Matcher.ConnString)
 	if err != nil {
@@ -68,6 +114,9 @@ func importAction(c *cli.Context) error {
 }
 
 func openInput(ctx context.Context, c *http.Client, n string) (io.ReadCloser, error) {
+	if n == "-" {
+		return os.Stdin, nil
+	}
 	f, ferr := os.Open(n)
 	if ferr == nil {
 		return f, nil