Backend related bugfixes

backend/storage/fs/store.go

@@ -61,6 +61,7 @@ func (s *Store) Exists(ctx context.Context, name string) bool {
 
 // List returns a list of all entities
 func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
+	out.Debug(ctx, "fs.List(%s)", prefix)
 	files := make([]string, 0, 100)
 	if err := filepath.Walk(s.path, func(path string, info os.FileInfo, err error) error {
 		if err != nil {

backend/storage/kv/consul/store.go

@@ -2,18 +2,27 @@ package consul
 
 import (
 	"context"
+	"strings"
 
 	"github.com/blang/semver"
 	api "github.com/hashicorp/consul/api"
+	"github.com/justwatchcom/gopass/utils/out"
 )
 
 // Store is a consul-backed store
 type Store struct {
-	api *api.Client
+	prefix string
+	api    *api.Client
 }
 
 // New creates a new consul store
-func New(host, datacenter, token string) (*Store, error) {
+func New(host, prefix, datacenter, token string) (*Store, error) {
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	if strings.HasPrefix(prefix, "/") {
+		prefix = strings.TrimPrefix(prefix, "/")
+	}
 	client, err := api.NewClient(&api.Config{
 		Address:    host,
 		Datacenter: datacenter,
@@ -23,12 +32,15 @@ func New(host, datacenter, token string) (*Store, error) {
 		return nil, err
 	}
 	return &Store{
-		api: client,
+		api:    client,
+		prefix: prefix,
 	}, nil
 }
 
 // Get retrieves a single entry
 func (s *Store) Get(ctx context.Context, name string) ([]byte, error) {
+	name = s.prefix + name
+	out.Debug(ctx, "consul.Get(%s)", name)
 	p, _, err := s.api.KV().Get(name, nil)
 	if err != nil {
 		return nil, err
@@ -41,6 +53,8 @@ func (s *Store) Get(ctx context.Context, name string) ([]byte, error) {
 
 // Set writes a single entry
 func (s *Store) Set(ctx context.Context, name string, value []byte) error {
+	name = s.prefix + name
+	out.Debug(ctx, "consul.Set(%s)", name)
 	p := &api.KVPair{
 		Key:   name,
 		Value: value,
@@ -51,12 +65,15 @@ func (s *Store) Set(ctx context.Context, name string, value []byte) error {
 
 // Delete removes a single entry
 func (s *Store) Delete(ctx context.Context, name string) error {
+	name = s.prefix + name
+	out.Debug(ctx, "consul.Delete(%s)", name)
 	_, err := s.api.KV().Delete(name, nil)
 	return err
 }
 
 // Exists checks if a given entry exists
 func (s *Store) Exists(ctx context.Context, name string) bool {
+	out.Debug(ctx, "consul.Exists(%s)", name)
 	v, err := s.Get(ctx, name)
 	if err == nil && v != nil {
 		return true
@@ -65,29 +82,41 @@ func (s *Store) Exists(ctx context.Context, name string) bool {
 }
 
 // List lists all entries matching the given prefix
-func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
+func (s *Store) List(ctx context.Context, _ string) ([]string, error) {
+	prefix := s.prefix
+	out.Debug(ctx, "consul.List(%s)", prefix)
 	pairs, _, err := s.api.KV().List(prefix, nil)
 	if err != nil {
 		return nil, err
 	}
 	res := make([]string, len(pairs))
 	for _, kvp := range pairs {
-		res = append(res, kvp.Key)
+		res = append(res, strings.TrimPrefix(kvp.Key, s.prefix))
 	}
 	return res, nil
 }
 
 // IsDir checks if the given entry is a directory
 func (s *Store) IsDir(ctx context.Context, name string) bool {
+	name = s.prefix + name
+	out.Debug(ctx, "consul.IsDir(%s)", name)
+	count := 0
 	ls, err := s.List(ctx, name)
-	if err == nil && len(ls) > 1 {
-		return true
+	if err != nil {
+		return false
 	}
-	return false
+	for _, e := range ls {
+		if strings.HasPrefix(e, name) {
+			count++
+		}
+	}
+	return count > 1
 }
 
 // Prune removes the given tree
 func (s *Store) Prune(ctx context.Context, prefix string) error {
+	prefix = s.prefix + prefix
+	out.Debug(ctx, "consul.Prune(%s)", prefix)
 	return s.Delete(ctx, prefix)
 }
 

docs/backends.md

@@ -12,13 +12,58 @@ All backends are in their own packages below `backend/`. They need to implement
 interfaces defined in the backend package and have their identification added to
 the context handlers in the same package.
 
-## Storage Backends (store)
+## Storage Backends (storage)
 
 ### Filesystem (fs)
 
 Right now there is only one storage backend implemented: Storing bytes on disk.
 
-## SCM Backends (sync)
+### In Memory (inmem)
+
+This is a volatile in-memory backend for tests.
+
+WARNING: All data is lost when gopass stops!
+
+### Consul (consul)
+
+This is an experimental storage backend that stores data in Consul.
+Make sure to either combine this with a crypto backend or make sure
+the data in Consul is properly protected as this backend does no
+encryption on it's own.
+
+#### Usage
+
+Until Consul support is fully integrated you need to manually setup a mount
+using the Consul backend.
+
+Add a new mount to your `config.yml` (usually at `.config/gopass/config.yml`):
+
+```bash
+cat <<EOF >> $HOME/.config/gopass/config.yml
+mounts:
+  consul:
+    path: plain-noop-consul+https://consul:8500/some/prefix/?token=some-token&datacenter=your-dc
+EOF
+```
+
+This will setup an unecrypted backend, i.e. your secrets in Consul will be only
+protected by Consul's ACLs and anyone who can access your Consul K/V prefix
+can read your secrets.
+
+You probably want to use a crypto backend to protect your secrets like in the
+following example:
+
+```bash
+gopass xc generate
+KEY=$(gopass xc list-private-keys | tail -1 | cut -d' ' -f1)
+gopass init --path='xc-noop-consul+https://consul:8500/foo/bar/?token=some-token&datacenter=you-dc' --store=consul --crypto=xc --sync=noop $KEY
+gopass mounts
+```
+
+## RCS Backends (rcs)
+
+These are revision control backends talking to difference source control
+management systems.
 
 ### CLI-based git (gitcli)
 
@@ -36,13 +81,13 @@ it unseable for most gopass usecases. However we still keep this backend around
 in case upstream manages to implement proper merges. In that case this will
 quickly become the default SCM backend.
 
-### Git Mock
+### Noop (noop)
 
 This is a no-op backend for testing SCM-less support.
 
 ## Crypto Backends (crypto)
 
-### CLI-based GPG
+### CLI-based GPG (gitcli)
 
 This backend is based on calling the gpg binary. This is the recommended backend
 since we believe that it's the most secure and one and it's compatible with
@@ -51,7 +96,7 @@ difficult to use, there are lot's of different versions being used and the
 output is not very machine readable. We will continue to support this backend
 in the future, but we'd like to to move to a different default backend if possible.
 
-### GPG Mock
+### Plaintext (plain)
 
 This is a no-op backend used for testing.
 
@@ -75,3 +120,6 @@ using existing building blocks - we're a little wary to recommend it for broader
 
 Also it requires it's own Keyring/Agent infrastructure as the keyformat is quite
 different from what GPG is using.
+
+Please see the backend [Readme](https://github.com/justwatchcom/gopass/blob/master/backend/crypto/xc/README.md) for more details. Proper documentation for this
+backend still needs to written and will be added at a later point.

store/root/mount.go

@@ -35,13 +35,16 @@ func (r *Store) addMount(ctx context.Context, alias, path string, sc *config.Sto
 		return errors.Errorf("%s is already mounted", alias)
 	}
 
+	out.Debug(ctx, "addMount - Path: %s - StoreConfig: %+v", path, sc)
 	// propagate our config settings to the sub store
 	if sc != nil {
 		if !backend.HasCryptoBackend(ctx) {
 			ctx = backend.WithCryptoBackend(ctx, sc.Path.Crypto)
+			out.Debug(ctx, "addMount - Using crypto backend %s", backend.CryptoBackendName(sc.Path.Crypto))
 		}
 		if !backend.HasRCSBackend(ctx) {
 			ctx = backend.WithRCSBackend(ctx, sc.Path.RCS)
+			out.Debug(ctx, "addMount - Using RCS backend %s", backend.RCSBackendName(sc.Path.RCS))
 		}
 	}
 	s, err := sub.New(ctx, alias, path, config.Directory())

store/root/store.go

@@ -37,20 +37,25 @@ func New(ctx context.Context, cfg *config.Config) (*Store, error) {
 	}
 
 	// create the base store
-	if !backend.HasCryptoBackend(ctx) {
-		ctx = backend.WithCryptoBackend(ctx, cfg.Root.Path.Crypto)
-	}
-	if !backend.HasRCSBackend(ctx) {
-		ctx = backend.WithRCSBackend(ctx, cfg.Root.Path.RCS)
-	}
-	if !backend.HasStorageBackend(ctx) {
-		ctx = backend.WithStorageBackend(ctx, cfg.Root.Path.Storage)
-	}
-	s, err := sub.New(ctx, "", r.Path(), config.Directory())
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to initialize the root store at '%s': %s", r.Path(), err)
+	{
+		// capture ctx to limit effect on the next sub.New call and to not
+		// propagate it's effects to the mounts below
+		ctx := ctx
+		if !backend.HasCryptoBackend(ctx) {
+			ctx = backend.WithCryptoBackend(ctx, cfg.Root.Path.Crypto)
+		}
+		if !backend.HasRCSBackend(ctx) {
+			ctx = backend.WithRCSBackend(ctx, cfg.Root.Path.RCS)
+		}
+		if !backend.HasStorageBackend(ctx) {
+			ctx = backend.WithStorageBackend(ctx, cfg.Root.Path.Storage)
+		}
+		s, err := sub.New(ctx, "", r.url.String(), config.Directory())
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to initialize the root store at '%s': %s", r.Path(), err)
+		}
+		r.store = s
 	}
-	r.store = s
 
 	// initialize all mounts
 	for alias, sc := range cfg.Mounts {

store/sub/init.go

@@ -53,7 +53,7 @@ You can add secondary stores with gopass init --path <path to secondary store> -
 	}
 
 	if err := s.saveRecipients(ctx, recipients, "Initialized Store for "+strings.Join(recipients, ", "), true); err != nil {
-		return errors.Errorf("failed to initialize store: %v", err)
+		return errors.Wrapf(err, "failed to initialize store: %s", err)
 	}
 
 	return nil

store/sub/list.go

@@ -10,7 +10,7 @@ var (
 	sep = string(filepath.Separator)
 )
 
-// List will list all entries in this.storage
+// List will list all entries in this store
 func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
 	lst, err := s.storage.List(ctx, prefix)
 	if err != nil {

store/sub/recipients.go

@@ -5,8 +5,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 
@@ -18,7 +16,6 @@ import (
 const (
 	keyDir    = ".public-keys"
 	oldKeyDir = ".gpg-keys"
-	dirMode   = 0700
 )
 
 // Recipients returns the list of recipients of this.storage
@@ -201,11 +198,6 @@ func (s *Store) saveRecipients(ctx context.Context, rs []string, msg string, exp
 		}
 	}
 
-	// save recipients' public keys
-	if err := os.MkdirAll(filepath.Join(s.url.Path, keyDir), dirMode); err != nil {
-		return errors.Wrapf(err, "failed to create key dir '%s'", keyDir)
-	}
-
 	// save all recipients public keys to the repo
 	if exportKeys {
 		if _, err := s.ExportMissingPublicKeys(ctx, rs); err != nil {

store/sub/store.go

@@ -38,10 +38,13 @@ type Store struct {
 
 // New creates a new store, copying settings from the given root store
 func New(ctx context.Context, alias, path string, cfgdir string) (*Store, error) {
+	// TODO
+	out.Debug(ctx, "Path: %s", path)
 	u, err := backend.ParseURL(path)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse path URL '%s': %s", path, err)
 	}
+	out.Debug(ctx, "URL: %s", u.String())
 
 	s := &Store{
 		alias:  alias,
@@ -53,6 +56,7 @@ func New(ctx context.Context, alias, path string, cfgdir string) (*Store, error)
 	// init store backend
 	if backend.HasStorageBackend(ctx) {
 		s.url.Storage = backend.GetStorageBackend(ctx)
+		out.Debug(ctx, "sub.New - Using storage backend from ctx: %s", backend.StorageBackendName(s.url.Storage))
 	}
 	if err := s.initStorageBackend(ctx); err != nil {
 		return nil, err
@@ -61,6 +65,7 @@ func New(ctx context.Context, alias, path string, cfgdir string) (*Store, error)
 	// init sync backend
 	if backend.HasRCSBackend(ctx) {
 		s.url.RCS = backend.GetRCSBackend(ctx)
+		out.Debug(ctx, "sub.New - Using RCS backend from ctx: %s", backend.RCSBackendName(s.url.RCS))
 	}
 	if err := s.initRCSBackend(ctx); err != nil {
 		return nil, err
@@ -69,6 +74,7 @@ func New(ctx context.Context, alias, path string, cfgdir string) (*Store, error)
 	// init crypto backend
 	if backend.HasCryptoBackend(ctx) {
 		s.url.Crypto = backend.GetCryptoBackend(ctx)
+		out.Debug(ctx, "sub.New - Using Crypto backend from ctx: %s", backend.CryptoBackendName(s.url.Crypto))
 	}
 	if err := s.initCryptoBackend(ctx); err != nil {
 		return nil, err
@@ -87,7 +93,7 @@ func (s *Store) initStorageBackend(ctx context.Context) error {
 		s.storage = inmem.New()
 	case backend.Consul:
 		out.Debug(ctx, "Using Storage Backend: consul")
-		store, err := kvconsul.New(s.url.Host+":"+s.url.Port, s.url.Query.Get("datacenter"), s.url.Query.Get("token"))
+		store, err := kvconsul.New(s.url.Host+":"+s.url.Port, s.url.Path, s.url.Query.Get("datacenter"), s.url.Query.Get("token"))
 		if err != nil {
 			return err
 		}