Merge Storage and RCS backends into the Storage backend (#1455)

Fixes #1454
Fixes #1457

RELEASE_NOTES=[CLEANUP] Merge Storage and RCS backends

Signed-off-by: Dominik Schulz <dominik.schulz@gauner.org>

docs/backends.md

@@ -1,28 +1,27 @@
 # Backends
 
-gopass supports pluggable backends for Storage (`store`), Encryption (`crypto`) and Source-Control-Management (`sync`).
+gopass supports pluggable backends for Storage and Revision Control System (`storage`) and Encryption (`crypto`).
 
 As of today, the names and responsibilities of these backends are still unstable and will probably change.
 
-By providing suitable backends, gopass can use different kinds of encryption (see XC below) or storage.
-For example, it is pretty straightforward to add mercurial or bazaar as an SCM backend or
-implement a Vault storage.
+By providing suitable backends, gopass can use different kinds of encryption or storage.
+For example, it is pretty straightforward to add mercurial or bazaar as an SCM backend.
 
 All backends are in their own packages below `backend/`. They need to implement the
 interfaces defined in the backend package and have their identification added to
 the context handlers in the same package.
 
-## Storage Backends (storage)
+## Storage and RCS Backends (storage)
 
 ### Filesystem (fs)
 
-This is the only stable storage backend. It stores the encrypted (see "Crypto Backends") data directly in the filesystem.
+This is the simplest storage backend. It stores the encrypted (see "Crypto Backends") data directly in the filesystem. It does not use version control and the relevant operations are no ops.
 
-### In Memory (inmem)
+### Filesystem with Git (gitfs)
 
-This is a volatile in-memory backend for tests.
-
-**WARNING**: All data is lost when gopass stops!
+This is the default storage backend. It stores the encrypted data directly in the
+filesystem. It uses an external git binary to provide history and remote sync
+operations.
 
 ### On Disk (ondisk)
 
@@ -43,21 +42,22 @@ The age keyring itself is also age encrypted serialized JSON.
 
 TODO: Add commands to decrypt an ondisk/age store without gopass.
 
-## RCS Backends (rcs)
-
-These are revision control backends talking to various source control
-management systems.
-
-### CLI-based git (gitcli)
+#### Background: How do access ondisk secrets without gopass
 
-The CLI-based git backend requires a properly configured git binary. It has the
-most features of all SCM backends and is pretty stable. One major drawback is that
-it sometimes fails if commit signing is enabled and the interaction with GPG
-fails.
+This section assumes `age` and `jq` are properly installed.
 
-### Noop (noop)
-
-This is a no-op backend for testing SCM-less support.
+```
+# Decrypt the gopass-age keyring
+age -d -o /tmp/keyring ~/.config/gopass/age-keyring.age
+# Extract the private key
+cat /tmp/keyring | jq ".[1].identity" | cut -d'"' -f2 > /tmp/private-key
+# Decrypt the index
+# TODO
+# Locate the latest secrets
+# TODO
+# Decrypt it
+age -d -i /tmp/private-key -o /tmp/plaintext ~/.local/share/gopass/stores/root/foo.age
+```
 
 ## Crypto Backends (crypto)
 
@@ -103,3 +103,4 @@ encrypted keyring on top (using age in scrypt password mode). It also has
 use their ssh public keys for age encryption.
 
 This backend might very well become the new default backend.
+

internal/action/action_test.go

@@ -24,9 +24,8 @@ func newMock(ctx context.Context, u *gptest.Unit) (*Action, error) {
 	cfg := config.Load()
 	cfg.Path = u.StoreDir("")
 
-	ctx = backend.WithRCSBackend(ctx, backend.Noop)
 	ctx = backend.WithCryptoBackend(ctx, backend.Plain)
-	ctx = backend.WithStorageBackend(ctx, backend.FS)
+	ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 	act, err := newAction(cfg, semver.Version{})
 	if err != nil {
 		return nil, err

internal/action/clone.go

@@ -24,9 +24,6 @@ func (s *Action) Clone(c *cli.Context) error {
 	if c.IsSet("crypto") {
 		ctx = backend.WithCryptoBackendString(ctx, c.String("crypto"))
 	}
-	if c.IsSet("sync") {
-		ctx = backend.WithRCSBackendString(ctx, c.String("sync"))
-	}
 
 	if c.Args().Len() < 1 {
 		return ExitError(ExitUsage, nil, "Usage: %s clone repo [mount]", s.Name)
@@ -43,11 +40,11 @@ func (s *Action) Clone(c *cli.Context) error {
 	return s.clone(ctx, repo, mount, path)
 }
 
-func rcsBackendOrDefault(ctx context.Context) backend.RCSBackend {
-	if be := backend.GetRCSBackend(ctx); be != backend.Noop {
+func storageBackendOrDefault(ctx context.Context) backend.StorageBackend {
+	if be := backend.GetStorageBackend(ctx); be != backend.FS {
 		return be
 	}
-	return backend.GitCLI
+	return backend.GitFS
 }
 
 func (s *Action) clone(ctx context.Context, repo, mount, path string) error {
@@ -71,7 +68,7 @@ func (s *Action) clone(ctx context.Context, repo, mount, path string) error {
 
 	// clone repo
 	debug.Log("Cloning repo '%s' to '%s'", repo, path)
-	if _, err := backend.CloneRCS(ctx, rcsBackendOrDefault(ctx), repo, path); err != nil {
+	if _, err := backend.Clone(ctx, storageBackendOrDefault(ctx), repo, path); err != nil {
 		return ExitError(ExitGit, err, "failed to clone repo '%s' to '%s': %s", repo, path, err)
 	}
 

internal/action/clone_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/gopasspw/gopass/internal/backend"
-	git "github.com/gopasspw/gopass/internal/backend/rcs/git/cli"
+	git "github.com/gopasspw/gopass/internal/backend/storage/gitfs"
 	"github.com/gopasspw/gopass/internal/config"
 	"github.com/gopasspw/gopass/internal/gptest"
 	"github.com/gopasspw/gopass/internal/out"
@@ -26,7 +26,7 @@ func aGitRepo(ctx context.Context, u *gptest.Unit, t *testing.T, name string) st
 	gd := filepath.Join(u.Dir, name)
 	assert.NoError(t, os.MkdirAll(gd, 0700))
 
-	_, err := git.Open(gd)
+	_, err := git.New(gd)
 	assert.Error(t, err)
 
 	idf := filepath.Join(gd, ".gpg-id")
@@ -46,7 +46,7 @@ func TestClone(t *testing.T) {
 	ctx := context.Background()
 	ctx = ctxutil.WithAlwaysYes(ctx, true)
 	ctx = ctxutil.WithInteractive(ctx, false)
-	ctx = backend.WithRCSBackend(ctx, backend.GitCLI)
+	ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 
 	act, err := newMock(ctx, u)
 	require.NoError(t, err)

internal/action/commands.go

@@ -82,11 +82,7 @@ func (s *Action) GetCommands() []*cli.Command {
 				},
 				&cli.StringFlag{
 					Name:  "crypto",
-					Usage: "Select crypto backend (gpg, gpgcli, plain, xc)",
-				},
-				&cli.StringFlag{
-					Name:  "sync",
-					Usage: "Select sync backend (git, gitcli, noop)",
+					Usage: "Select crypto backend (gpgcli, age, plain, xc)",
 				},
 			},
 		},
@@ -268,7 +264,6 @@ func (s *Action) GetCommands() []*cli.Command {
 				"must be a file and the other one a secret. If you want the source to " +
 				"be securely removed after copying, use 'gopass binary move'",
 			Before:       s.Initialized,
-			Aliases:      []string{"cp"},
 			Action:       s.BinaryCopy,
 			BashComplete: s.Complete,
 			Hidden:       true,
@@ -292,7 +287,6 @@ func (s *Action) GetCommands() []*cli.Command {
 				"and validated. If you don't want the source to be removed use " +
 				"'gopass binary copy'",
 			Before:       s.Initialized,
-			Aliases:      []string{"mv"},
 			Action:       s.BinaryMove,
 			BashComplete: s.Complete,
 			Hidden:       true,

internal/action/config.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/gopasspw/gopass/internal/backend/storage/kv/ondisk"
+	"github.com/gopasspw/gopass/internal/backend/storage/ondisk"
 	"github.com/gopasspw/gopass/internal/debug"
 	"github.com/gopasspw/gopass/internal/out"
 	"github.com/gopasspw/gopass/pkg/ctxutil"

internal/action/convert.go

@@ -14,7 +14,6 @@ func (s *Action) Convert(c *cli.Context) error {
 	move := c.Bool("move")
 	storage := backend.StorageBackendFromName(c.String("storage"))
 	crypto := backend.CryptoBackendFromName(c.String("crypto"))
-	rcs := backend.RcsBackendFromName(c.String("rcs"))
 
-	return s.Store.Convert(ctx, store, crypto, rcs, storage, move)
+	return s.Store.Convert(ctx, store, crypto, storage, move)
 }

internal/action/history_test.go

@@ -29,9 +29,8 @@ func TestHistory(t *testing.T) {
 
 	ctx := context.Background()
 	ctx = ctxutil.WithAlwaysYes(ctx, true)
-	ctx = backend.WithRCSBackend(ctx, backend.GitCLI)
 	ctx = backend.WithCryptoBackend(ctx, backend.Plain)
-	ctx = backend.WithStorageBackend(ctx, backend.FS)
+	ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 
 	cfg := config.New()
 	cfg.Path = u.StoreDir("")

internal/action/init.go

@@ -81,9 +81,6 @@ func initParseContext(ctx context.Context, c *cli.Context) context.Context {
 	if c.IsSet("crypto") {
 		ctx = backend.WithCryptoBackendString(ctx, c.String("crypto"))
 	}
-	if c.IsSet("rcs") {
-		ctx = backend.WithRCSBackendString(ctx, c.String("rcs"))
-	}
 	if c.IsSet("storage") {
 		ctx = backend.WithStorageBackendString(ctx, c.String("storage"))
 	}
@@ -92,13 +89,9 @@ func initParseContext(ctx context.Context, c *cli.Context) context.Context {
 		debug.Log("Using default Crypto Backend (GPGCLI)")
 		ctx = backend.WithCryptoBackend(ctx, backend.GPGCLI)
 	}
-	if !backend.HasRCSBackend(ctx) {
-		debug.Log("Using default RCS backend (GitCLI)")
-		ctx = backend.WithRCSBackend(ctx, backend.GitCLI)
-	}
 	if !backend.HasStorageBackend(ctx) {
-		debug.Log("Using default storage backend (FS)")
-		ctx = backend.WithStorageBackend(ctx, backend.FS)
+		debug.Log("Using default storage backend (GitFS)")
+		ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 	}
 
 	ctx = out.WithPrefix(ctx, "[init] ")
@@ -144,8 +137,8 @@ func (s *Action) init(ctx context.Context, alias, path string, keys ...string) e
 		}
 	}
 
-	if backend.HasRCSBackend(ctx) {
-		bn := backend.RCSBackendName(backend.GetRCSBackend(ctx))
+	if backend.HasStorageBackend(ctx) {
+		bn := backend.StorageBackendName(backend.GetStorageBackend(ctx))
 		debug.Log("Initializing RCS (%s) ...", bn)
 		if err := s.rcsInit(ctx, alias, ctxutil.GetUsername(ctx), ctxutil.GetEmail(ctx)); err != nil {
 			debug.Log("Stacktrace: %+v\n", err)

internal/action/init_test.go

@@ -25,7 +25,6 @@ func TestInit(t *testing.T) {
 	ctx := context.Background()
 	ctx = ctxutil.WithAlwaysYes(ctx, true)
 	ctx = ctxutil.WithInteractive(ctx, false)
-	ctx = backend.WithRCSBackend(ctx, backend.Noop)
 	ctx = backend.WithCryptoBackend(ctx, backend.Plain)
 	ctx = backend.WithStorageBackend(ctx, backend.FS)
 
@@ -87,20 +86,10 @@ func TestInitParseContext(t *testing.T) {
 				return nil
 			},
 		},
-		{
-			name:  "rcs noop",
-			flags: map[string]string{"rcs": "noop"},
-			check: func(ctx context.Context) error {
-				if be := backend.GetRCSBackend(ctx); be != backend.Noop {
-					return fmt.Errorf("wrong backend: %d", be)
-				}
-				return nil
-			},
-		},
 		{
 			name: "default",
 			check: func(ctx context.Context) error {
-				if backend.GetRCSBackend(ctx) != backend.GitCLI {
+				if backend.GetStorageBackend(ctx) != backend.GitFS {
 					return fmt.Errorf("wrong backend")
 				}
 				return nil

internal/action/rcs.go

@@ -22,11 +22,11 @@ func (s *Action) RCSInit(c *cli.Context) error {
 	store := c.String("store")
 	un := termio.DetectName(c.Context, c)
 	ue := termio.DetectEmail(c.Context, c)
-	ctx = backend.WithRCSBackendString(ctx, c.String("rcs"))
+	ctx = backend.WithStorageBackendString(ctx, c.String("storage"))
 
 	// default to git
-	if !backend.HasRCSBackend(ctx) {
-		ctx = backend.WithRCSBackend(ctx, backend.GitCLI)
+	if !backend.HasStorageBackend(ctx) {
+		ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 	}
 
 	if err := s.rcsInit(ctx, store, un, ue); err != nil {
@@ -36,7 +36,11 @@ func (s *Action) RCSInit(c *cli.Context) error {
 }
 
 func (s *Action) rcsInit(ctx context.Context, store, un, ue string) error {
-	bn := backend.RCSBackendName(backend.GetRCSBackend(ctx))
+	// TODO this is a hack
+	if backend.GetStorageBackend(ctx) == backend.FS {
+		return nil
+	}
+	bn := backend.StorageBackendName(backend.GetStorageBackend(ctx))
 	out.Green(ctx, "Initializing git repository (%s) for %s / %s...", bn, un, ue)
 
 	userName, userEmail := s.getUserData(ctx, store, un, ue)

internal/action/recipients.go

@@ -100,7 +100,7 @@ func (s *Action) RecipientsAdd(c *cli.Context) error {
 			}
 			keys = []string{r}
 		}
-		if len(keys) < 1 && !force {
+		if len(keys) < 1 && !force && crypto.Name() == "gpgcli" {
 			out.Cyan(ctx, "Warning: No matching valid key found. If the key is in your keyring you may need to validate it.")
 			out.Cyan(ctx, "If this is your key: gpg --edit-key %s; trust (set to ultimate); quit", r)
 			out.Cyan(ctx, "If this is not your key: gpg --edit-key %s; lsign; trust; save; quit", r)

internal/action/recipients_test.go

@@ -72,7 +72,7 @@ gopass
 
 	t.Run("add recipient 0xBEEFFEED", func(t *testing.T) {
 		defer buf.Reset()
-		assert.Error(t, act.RecipientsAdd(gptest.CliCtx(ctx, t, "0xBEEFFEED")))
+		assert.NoError(t, act.RecipientsAdd(gptest.CliCtx(ctx, t, "0xBEEFFEED")))
 	})
 
 	t.Run("remove recipient 0xDEADBEEF", func(t *testing.T) {

internal/action/sync.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/gopasspw/gopass/internal/backend/rcs/noop"
 	"github.com/gopasspw/gopass/internal/debug"
 	"github.com/gopasspw/gopass/internal/notify"
 	"github.com/gopasspw/gopass/internal/out"
@@ -89,11 +88,11 @@ func (s *Action) syncMount(ctx context.Context, mp string) error {
 		numMP = len(l)
 	}
 
-	if sub.RCS().Name() == noop.New().Name() {
-		out.Error(ctxno, "\n   WARNING: Mount uses RCS backend 'noop'. Not syncing!\n")
+	if sub.Storage().Name() == "fs" {
+		out.Yellow(ctxno, "\n   WARNING: Mount uses Storage backend 'fs'. Not syncing!\n")
 	} else {
 		out.Print(ctxno, "\n   "+color.GreenString("git pull and push ... "))
-		if err := sub.RCS().Push(ctx, "", ""); err != nil {
+		if err := sub.Storage().Push(ctx, "", ""); err != nil {
 			if errors.Cause(err) == store.ErrGitNoRemote {
 				out.Yellow(ctx, "Skipped (no remote)")
 				debug.Log("Failed to push '%s' to its remote: %s", name, err)
@@ -144,7 +143,7 @@ func (s *Action) syncMount(ctx context.Context, mp string) error {
 
 	// only run second push if we did export any keys
 	if exported {
-		if err := sub.RCS().Push(ctx, "", ""); err != nil {
+		if err := sub.Storage().Push(ctx, "", ""); err != nil {
 			out.Error(ctx, "Failed to push '%s' to its remote: %s", name, err)
 			return err
 		}

internal/action/templates_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/gopasspw/gopass/pkg/ctxutil"
 
 	_ "github.com/gopasspw/gopass/internal/backend/crypto"
-	_ "github.com/gopasspw/gopass/internal/backend/rcs"
 	_ "github.com/gopasspw/gopass/internal/backend/storage"
 
 	"github.com/fatih/color"

internal/action/unclip_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	_ "github.com/gopasspw/gopass/internal/backend/crypto"
-	_ "github.com/gopasspw/gopass/internal/backend/rcs"
 	_ "github.com/gopasspw/gopass/internal/backend/storage"
 )
 

internal/action/update_test.go

@@ -18,7 +18,6 @@ import (
 	"github.com/gopasspw/gopass/pkg/ctxutil"
 
 	_ "github.com/gopasspw/gopass/internal/backend/crypto"
-	_ "github.com/gopasspw/gopass/internal/backend/rcs"
 	_ "github.com/gopasspw/gopass/internal/backend/storage"
 
 	"github.com/dominikschulz/github-releases/ghrel"