Add Vault backend

Fixes #725

.gitignore

@@ -47,3 +47,5 @@ dist/
 # go-fuzz
 *-fuzz.zip
 workdir/
+
+NOTICE.new

Makefile

@@ -25,7 +25,7 @@ OK := $(shell tput setaf 6; echo ' [OK]'; tput sgr0;)
 all: build completion
 build: $(GOPASS_OUTPUT)
 completion: $(BASH_COMPLETION_OUTPUT) $(FISH_COMPLETION_OUTPUT) $(ZSH_COMPLETION_OUTPUT)
-travis: sysinfo crosscompile build install test codequality completion
+travis: sysinfo crosscompile build install legal test codequality completion
 
 sysinfo:
 	@echo ">> SYSTEM INFORMATION"
@@ -117,6 +117,16 @@ install-completion: completion
 	@install -m 0755 $(FISH_COMPLETION_OUTPUT) $(DESTDIR)$(PREFIX)/share/fish/vendor_completions.d/gopass.fish
 	@printf '%s\n' '$(OK)'
 
+legal:
+	@echo ">> LEGAL"
+	@echo -n "   LICENSES   "
+	@which fossa > /dev/null; if [ $$? -ne 0 ]; then \
+		$(GO) get -u go get -u github.com/pmezard/licenses; \
+	fi
+	@licenses . > NOTICE.new
+	@diff NOTICE.txt NOTICE.new || exit 1
+	@printf '%s\n' '$(OK)'
+
 codequality:
 	@echo ">> CODE QUALITY"
 	@echo -n "     FMT       "

NOTICE.txt

@@ -0,0 +1,54 @@
+github.com/justwatchcom/gopass                                                         MIT License
+github.com/justwatchcom/gopass/vendor/github.com/atotto/clipboard                      BSD 3-clause "New" or "Revised" License (97%)
+github.com/justwatchcom/gopass/vendor/github.com/blang/semver                          MIT License
+github.com/justwatchcom/gopass/vendor/github.com/cenkalti/backoff                      MIT License
+github.com/justwatchcom/gopass/vendor/github.com/emirpasic/gods                        BSD 2-clause "Simplified" License (90%)
+github.com/justwatchcom/gopass/vendor/github.com/fatih/color                           MIT License
+github.com/justwatchcom/gopass/vendor/github.com/fatih/structs                         MIT License
+github.com/justwatchcom/gopass/vendor/github.com/gdamore/encoding                      Apache License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/gdamore/tcell                         Apache License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/godbus/dbus                           BSD 2-clause "Simplified" License
+github.com/justwatchcom/gopass/vendor/github.com/gokyle/twofactor                      MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/golang/protobuf/proto                 BSD 3-clause "New" or "Revised" License (92%)
+github.com/justwatchcom/gopass/vendor/github.com/golang/snappy                         BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/consul/api                  Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/errwrap                     Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/go-cleanhttp                Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/go-multierror               Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/go-rootcerts                Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/hcl                         Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/serf/coordinate             Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/hashicorp/vault                       Mozilla Public License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/jbenet/go-context/io                  MIT License
+github.com/justwatchcom/gopass/vendor/github.com/jroimartin/gocui                      BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/github.com/kballard/go-shellquote                MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/kevinburke/ssh_config                 MIT License (93%)
+github.com/justwatchcom/gopass/vendor/github.com/lucasb-eyer/go-colorful               MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/martinhoefling/goxkcdpwgen/xkcdpwgen  MIT License
+github.com/justwatchcom/gopass/vendor/github.com/mattn/go-colorable                    MIT License
+github.com/justwatchcom/gopass/vendor/github.com/mattn/go-isatty                       MIT License (95%)
+github.com/justwatchcom/gopass/vendor/github.com/mattn/go-runewidth                    MIT License
+github.com/justwatchcom/gopass/vendor/github.com/mitchellh/go-homedir                  MIT License
+github.com/justwatchcom/gopass/vendor/github.com/mitchellh/mapstructure                MIT License
+github.com/justwatchcom/gopass/vendor/github.com/muesli/crunchy                        MIT License
+github.com/justwatchcom/gopass/vendor/github.com/muesli/goprogressbar                  MIT License
+github.com/justwatchcom/gopass/vendor/github.com/nsf/termbox-go                        MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/pkg/errors                            BSD 2-clause "Simplified" License
+github.com/justwatchcom/gopass/vendor/github.com/ryanuber/go-glob                      MIT License
+github.com/justwatchcom/gopass/vendor/github.com/schollz/closestmatch                  MIT License
+github.com/justwatchcom/gopass/vendor/github.com/sergi/go-diff/diffmatchpatch          MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/sethgrid/pester                       MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/skip2/go-qrcode                       MIT License (98%)
+github.com/justwatchcom/gopass/vendor/github.com/src-d/gcfg                            BSD 3-clause "New" or "Revised" License (97%)
+github.com/justwatchcom/gopass/vendor/github.com/urfave/cli                            MIT License
+github.com/justwatchcom/gopass/vendor/github.com/xanzy/ssh-agent                       Apache License 2.0
+github.com/justwatchcom/gopass/vendor/github.com/xrash/smetrics                        MIT License
+github.com/justwatchcom/gopass/vendor/golang.org/x/crypto                              BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/golang.org/x/net                                 BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/golang.org/x/sys/unix                            BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/golang.org/x/text                                BSD 3-clause "New" or "Revised" License (96%)
+github.com/justwatchcom/gopass/vendor/gopkg.in/src-d/go-billy.v4                       Apache License 2.0
+github.com/justwatchcom/gopass/vendor/gopkg.in/src-d/go-git.v4                         Apache License 2.0
+github.com/justwatchcom/gopass/vendor/gopkg.in/warnings.v0                             BSD 2-clause "Simplified" License (97%)
+github.com/justwatchcom/gopass/vendor/gopkg.in/yaml.v2                                 Apache License 2.0
+github.com/justwatchcom/gopass/vendor/rsc.io/qr                                        BSD 3-clause "New" or "Revised" License (96%)

docs/backends.md

@@ -123,3 +123,39 @@ different from what GPG is using.
 
 Please see the backend [Readme](https://github.com/justwatchcom/gopass/blob/master/pkg/backend/crypto/xc/README.md) for more details. Proper documentation for this
 backend still needs to written and will be added at a later point.
+
+### Vault (vault)
+
+This is an experimental crypto and storage backend currently available as a
+preview. This backend is special in that it's not implemented as a traditional
+backend but instead as an alternative `sub store` implementation. That was
+necessary as Vault already works with complex Secrets by itself and it didn't
+seem wise to force the internal gopass architecture onto this sophisticated
+storage scheme. That would have worked well for gopass, but would have stopped
+interoperability with other Vault users.
+
+**Note**: This backend fully relies on Vault for encryption and access
+management. It mostly exists as an easy access path to sync static secrets
+between a password store and Vault.
+
+To use the Vault backend manually create a mount in the config like in the
+following example:
+
+```
+cat <<EOF >> $HOME/.config/gopass/config.yml
+mounts:
+  vault:
+    path: vault+https://vault:8200/secret?token=some-token
+EOF
+```
+
+All `TLSConfig` options for Vault are supported as query parameters.
+
+| **Query Parameter** | **TLSConfig Attribute** | Description |
+| ------------------- | ----------------------- | ----------- |
+| tls-cacert | CACert | the Path to a PEM-encoded CA cert file |
+| tls-capath | CAPath | the Path to a directory of PEM-encoded CA cert files |
+| tls-clientcert | ClientCert | the Path to the certificate for Vault communication |
+| tls-clientkey | ClientKey | the path to the private key for Vault communication |
+| tls-servername | TLSServerName | set the SNI host when connecting |
+| tls-insecure | Disables SSL verification |

pkg/backend/crypto.go

@@ -18,6 +18,8 @@ const (
 	XC
 	// OpenPGP is a GPG1.x compatible pure-Go crypto backend
 	OpenPGP
+	// Vault is Hashicorp Vault backend
+	Vault
 )
 
 func (c CryptoBackend) String() string {

pkg/backend/strings.go

@@ -6,6 +6,7 @@ var (
 		"gpgcli":  GPGCLI,
 		"xc":      XC,
 		"openpgp": OpenPGP,
+		"vault":   Vault,
 	}
 	cryptoBackendToNameMap = map[CryptoBackend]string{}
 	rcsNameToBackendMap    = map[string]RCSBackend{

pkg/backend/url.go

@@ -139,8 +139,11 @@ func splitBackends(in string) (string, string, string, string, error) {
 	backends := p[0]
 	scheme := p[1]
 	p = strings.Split(backends, "-")
-	if len(p) < 3 {
+	if len(p) < 1 {
 		return "", "", "", "", fmt.Errorf("invalid")
 	}
+	if len(p) < 3 {
+		return p[0], "", "", scheme, nil
+	}
 	return p[0], p[1], p[2], scheme, nil
 }

pkg/store/root/init.go

@@ -26,7 +26,12 @@ func (r *Store) Initialized(ctx context.Context) bool {
 // Init tries to initialize a new password store location matching the object
 func (r *Store) Init(ctx context.Context, alias, path string, ids ...string) error {
 	out.Debug(ctx, "Instantiating new sub store %s at %s for %+v", alias, path, ids)
-	sub, err := sub.New(ctx, alias, path, r.cfg.Directory(), r.agent)
+	// parse backend URL
+	pathURL, err := backend.ParseURL(path)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse backend URL '%s': %s", path, err)
+	}
+	sub, err := sub.New(ctx, alias, pathURL, r.cfg.Directory(), r.agent)
 	if err != nil {
 		return err
 	}
@@ -94,7 +99,11 @@ func (r *Store) initialize(ctx context.Context) error {
 		if !backend.HasStorageBackend(ctx) {
 			ctx = backend.WithStorageBackend(ctx, r.cfg.Root.Path.Storage)
 		}
-		s, err := sub.New(ctx, "", r.url.String(), r.cfg.Directory(), r.agent)
+		bu, err := backend.ParseURL(r.url.String())
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse backend URL '%s': %s", r.url.String(), err)
+		}
+		s, err := sub.New(ctx, "", bu, r.cfg.Directory(), r.agent)
 		if err != nil {
 			return errors.Wrapf(err, "failed to initialize the root store at '%s': %s", r.url.String(), err)
 		}

pkg/store/root/mount.go

@@ -11,6 +11,7 @@ import (
 	"github.com/justwatchcom/gopass/pkg/out"
 	"github.com/justwatchcom/gopass/pkg/store"
 	"github.com/justwatchcom/gopass/pkg/store/sub"
+	"github.com/justwatchcom/gopass/pkg/store/vault"
 	"github.com/pkg/errors"
 )
 
@@ -47,33 +48,23 @@ func (r *Store) addMount(ctx context.Context, alias, path string, sc *config.Sto
 			out.Debug(ctx, "addMount - Using RCS backend %s", backend.RCSBackendName(sc.Path.RCS))
 		}
 	}
-	s, err := sub.New(ctx, alias, path, r.cfg.Directory(), r.agent)
+
+	// parse backend URL
+	pathURL, err := backend.ParseURL(path)
 	if err != nil {
-		return errors.Wrapf(err, "failed to initialize store '%s' at '%s': %s", alias, path, err)
+		return errors.Wrapf(err, "failed to parse backend URL '%s': %s", path, err)
 	}
 
-	if !s.Initialized(ctx) {
-		out.Debug(ctx, "[%s] Mount %s is not initialized", alias, path)
-		if len(keys) < 1 {
-			return errors.Errorf("password store %s is not initialized. Try gopass init --store %s --path %s", alias, alias, path)
-		}
-		if err := s.Init(ctx, path, keys...); err != nil {
-			return errors.Wrapf(err, "failed to initialize store '%s' at '%s'", alias, path)
-		}
-		out.Green(ctx, "Password store %s initialized for:", path)
-		for _, r := range s.Recipients(ctx) {
-			color.Yellow(r)
-		}
+	// initialize sub store
+	s, err := r.initSub(ctx, alias, pathURL, keys)
+	if err != nil {
+		return errors.Wrapf(err, "failed to init sub store")
 	}
 
 	r.mounts[alias] = s
 	if r.cfg.Mounts == nil {
 		r.cfg.Mounts = make(map[string]*config.StoreConfig, 1)
 	}
-	pathURL, err := backend.ParseURL(path)
-	if err != nil {
-		return errors.Wrapf(err, "failed to parse backend URL '%s': %s", path, err)
-	}
 	if sc == nil {
 		// imporant: copy root config to avoid overwriting it with sub store
 		// values
@@ -91,9 +82,47 @@ func (r *Store) addMount(ctx context.Context, alias, path string, sc *config.Sto
 		sc.Path.Storage = backend.GetStorageBackend(ctx)
 	}
 	r.cfg.Mounts[alias] = sc
+
+	out.Debug(ctx, "Added mount %s -> %s", alias, sc.Path.String())
 	return nil
 }
 
+func (r *Store) initSubVault(ctx context.Context, alias string, path *backend.URL) (store.Store, error) {
+	return vault.New(alias, path)
+}
+
+func (r *Store) initSub(ctx context.Context, alias string, path *backend.URL, keys []string) (store.Store, error) {
+	// init vault sub store
+	if backend.GetCryptoBackend(ctx) == backend.Vault || path.Crypto == backend.Vault {
+		out.Debug(ctx, "Initializing Vault Store at %s -> %s", alias, path.String())
+		return r.initSubVault(ctx, alias, path)
+	}
+
+	// init regular sub store
+	s, err := sub.New(ctx, alias, path, r.cfg.Directory(), r.agent)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to initialize store '%s' at '%s': %s", alias, path, err)
+	}
+
+	if s.Initialized(ctx) {
+		return s, nil
+	}
+
+	out.Debug(ctx, "[%s] Mount %s is not initialized", alias, path)
+	if len(keys) < 1 {
+		return s, errors.Errorf("password store %s is not initialized. Try gopass init --store %s --path %s", alias, alias, path)
+	}
+	if err := s.Init(ctx, path.String(), keys...); err != nil {
+		return s, errors.Wrapf(err, "failed to initialize store '%s' at '%s'", alias, path)
+	}
+	out.Green(ctx, "Password store %s initialized for:", path)
+	for _, r := range s.Recipients(ctx) {
+		color.Yellow(r)
+	}
+
+	return s, nil
+}
+
 // RemoveMount removes and existing mount
 func (r *Store) RemoveMount(ctx context.Context, alias string) error {
 	if _, found := r.mounts[alias]; !found {

pkg/store/sub/recipients_test.go

@@ -239,7 +239,7 @@ func TestListRecipients(t *testing.T) {
 	s, err := New(
 		ctx,
 		"",
-		tempdir,
+		backend.FromPath(tempdir),
 		tempdir,
 		nil,
 	)

pkg/store/sub/store.go

@@ -28,12 +28,7 @@ type Store struct {
 }
 
 // New creates a new store, copying settings from the given root store
-func New(ctx context.Context, alias, path, cfgdir string, agent *client.Client) (*Store, error) {
-	out.Debug(ctx, "sub.New - Path: %s", path)
-	u, err := backend.ParseURL(path)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse path URL '%s': %s", path, err)
-	}
+func New(ctx context.Context, alias string, u *backend.URL, cfgdir string, agent *client.Client) (*Store, error) {
 	out.Debug(ctx, "sub.New - URL: %s", u.String())
 
 	s := &Store{

pkg/store/sub/store_test.go

@@ -51,7 +51,7 @@ func createSubStore(dir string) (*Store, error) {
 	return New(
 		ctx,
 		"",
-		sd,
+		backend.FromPath(sd),
 		sd,
 		nil,
 	)
@@ -185,7 +185,7 @@ func TestNew(t *testing.T) {
 			ok:  false,
 		},
 	} {
-		s, err := New(tc.ctx, "", tempdir, tempdir, nil)
+		s, err := New(tc.ctx, "", backend.FromPath(tempdir), tempdir, nil)
 		if tc.ok {
 			assert.NoError(t, err)
 			assert.NotNil(t, s)

pkg/store/sub/templates_test.go

@@ -30,7 +30,7 @@ func TestTemplates(t *testing.T) {
 	s, err := New(
 		ctx,
 		"",
-		tempdir,
+		backend.FromPath(tempdir),
 		tempdir,
 		nil,
 	)