Support glob patterns in the configuration file (#209)

* Bail out if multiple config entries match a single location

If we have more than one entry in the configuration file that match the
location string that we are currently using (e.g. one with a trailing
slash and another without), we will end up choosing one configuration at
random.

When this situation arise, we should instead abort the execution to
avoid unexpected results.

Signed-off-by: Ludovico de Nittis <ludovico.denittis@collabora.com>

* Support glob patterns in the configuration file

This makes the configuration file more flexible.

Closes: #208

Signed-off-by: Ludovico de Nittis <ludovico.denittis@collabora.com>

README.md

@@ -241,8 +241,8 @@ Available configuration values:
 
 - `http-timeout` *DEPRECATED, see `store-options.<Location>.timeout`* - HTTP request timeout used in HTTP stores (not S3) in nanoseconds
 - `http-error-retry` *DEPRECATED, see `store-options.<Location>.error-retry` - Number of times to retry failed chunk requests from HTTP stores
-- `s3-credentials` - Defines credentials for use with S3 stores. Especially useful if more than one S3 store is used. The key in the config needs to be the URL scheme and host used for the store, excluding the path, but including the port number if used in the store URL. It is also possible to use a [standard aws credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) in order to store s3 credentials.
-- `store-options` - Allows customization of chunk and index stores, for example comression settings, timeouts, retry behavior and keys. Not all options are applicable to every store, some of these like `timeout` are ignored for local stores. Some of these options, such as the client certificates are overwritten with any values set in the command line. Note that the store location used in the command line needs to match the key under `store-options` exactly for these options to be used. Watch out for trailing `/` in URLs.
+- `s3-credentials` - Defines credentials for use with S3 stores. Especially useful if more than one S3 store is used. The key in the config needs to be the URL scheme and host used for the store, excluding the path, but including the port number if used in the store URL. The key can also contain glob patterns, and the available wildcards are `*`, `?` and `[…]`. Please refer to the [filepath.Match](https://pkg.go.dev/path/filepath#Match) documentation for additional information. It is also possible to use a [standard aws credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) in order to store s3 credentials.
+- `store-options` - Allows customization of chunk and index stores, for example compression settings, timeouts, retry behavior and keys. Not all options are applicable to every store, some of these like `timeout` are ignored for local stores. Some of these options, such as the client certificates are overwritten with any values set in the command line. Note that the store location used in the command line needs to match the key under `store-options` exactly for these options to be used. As for the `s3-credentials`, glob patterns are also supported. A configuration file where more than one key matches a single store location, is considered invalid.
   - `timeout` - Time limit for chunk read or write operation in nanoseconds. Default: 1 minute. If set to a negative value, timeout is infinite.
   - `error-retry` - Number of times to retry failed chunk requests. Default: 0.
   - `error-retry-base-interval` - Number of nanoseconds to wait before first retry attempt. Retry attempt number N for the same request will wait N times this interval. Default: 0.
@@ -285,6 +285,9 @@ Available configuration values:
     "https://10.0.0.1/": {
       "http-auth": "Bearer abcabcabc"
     },
+    "https://example.com/*/*/": {
+      "http-auth": "Bearer dXNlcjpwYXNzd29yZA=="
+    },
     "/path/to/local/cache": {
       "uncompressed": true
     }

cmd/desync/config.go

@@ -70,14 +70,21 @@ func (c Config) GetS3CredentialsFor(u *url.URL) (*credentials.Credentials, strin
 }
 
 // GetStoreOptionsFor returns optional config options for a specific store. Note that
-// the location string in the config file needs to match exactly (watch for trailing /).
-func (c Config) GetStoreOptionsFor(location string) desync.StoreOptions {
+// an error will be returned if the location string matches multiple entries in the
+// config file.
+func (c Config) GetStoreOptionsFor(location string) (options desync.StoreOptions, err error) {
+	found := false
+	options = desync.StoreOptions{}
 	for k, v := range c.StoreOptions {
 		if locationMatch(k, location) {
-			return v
+			if found {
+				return options, fmt.Errorf("multiple configuration entries match the location %q", location)
+			}
+			found = true
+			options = v
 		}
 	}
-	return desync.StoreOptions{}
+	return options, nil
 }
 
 func newConfigCommand(ctx context.Context) *cobra.Command {

cmd/desync/config_test.go

@@ -24,10 +24,33 @@ func TestConfigFile(t *testing.T) {
 	initConfig()
 
 	// If everything worked, the options should be set according to the config file created above
-	opt := cfg.GetStoreOptionsFor("/path/to/store")
+	opt, err := cfg.GetStoreOptionsFor("/path/to/store")
+	require.NoError(t, err)
 	require.True(t, opt.Uncompressed)
 
 	// The options for a non-matching store should be default
-	opt = cfg.GetStoreOptionsFor("/path/other-store")
+	opt, err = cfg.GetStoreOptionsFor("/path/other-store")
+	require.NoError(t, err)
 	require.False(t, opt.Uncompressed)
 }
+
+func TestConfigFileMultipleMatches(t *testing.T) {
+	cfgFileContent := []byte(`{"store-options": {"/path/to/store/":{"uncompressed": true}, "/path/to/store":{"uncompressed": false}}}`)
+	f, err := ioutil.TempFile("", "")
+	require.NoError(t, err)
+	f.Close()
+	defer os.Remove(f.Name())
+	require.NoError(t, ioutil.WriteFile(f.Name(), cfgFileContent, 0644))
+
+	// Set the global config file name
+	cfgFile = f.Name()
+
+	// Call init, this should use the custom config file and global "cfg" should contain the
+	// values
+	initConfig()
+
+	// We expect this to fail because both "/path/to/store/" and "/path/to/store" matches the
+	// provided location
+	_, err = cfg.GetStoreOptionsFor("/path/to/store")
+	require.Error(t, err)
+}

cmd/desync/location.go

@@ -2,36 +2,42 @@ package main
 
 import (
 	"net/url"
-	"path"
 	"path/filepath"
+	"strings"
 )
 
 // Returns true if the two locations are equal. Locations can be URLs or local file paths.
 // It can handle Unix as well as Windows paths. Example
 // http://host/path/ is equal http://host/path (no trailing /) and /tmp/path is
 // equal \tmp\path on Windows.
-func locationMatch(loc1, loc2 string) bool {
-	u1, _ := url.Parse(loc1)
-	u2, _ := url.Parse(loc2)
-	// See if we have at least one URL, Windows drive letters come out as single-letter
-	// scheme so we need more here.
-	if len(u1.Scheme) > 1 || len(u2.Scheme) > 1 {
-		if u1.Scheme != u2.Scheme || u1.Host != u2.Host {
-			return false
-		}
-		// URL paths should only use /, use path (not filepath) package to clean them
-		// before comparing
-		return path.Clean(u1.Path) == path.Clean(u2.Path)
+func locationMatch(pattern, loc string) bool {
+	l, err := url.Parse(loc)
+	if err != nil {
+		return false
+	}
+
+	// See if we have a URL, Windows drive letters come out as single-letter
+	// scheme, so we need more here.
+	if len(l.Scheme) > 1 {
+		// URL paths should only use / as separator, remove the trailing one, if any
+		trimmedLoc := strings.TrimSuffix(loc, "/")
+		trimmedPattern := strings.TrimSuffix(pattern, "/")
+		m, _ := filepath.Match(trimmedPattern, trimmedLoc)
+		return m
 	}
 
-	// We're dealing with two paths.
-	p1, err := filepath.Abs(loc1)
+	// We're dealing with a path.
+	p1, err := filepath.Abs(pattern)
+	if err != nil {
+		return false
+	}
+	p2, err := filepath.Abs(loc)
 	if err != nil {
 		return false
 	}
-	p2, err := filepath.Abs(loc2)
+	m, err := filepath.Match(p1, p2)
 	if err != nil {
 		return false
 	}
-	return p1 == p2
+	return m
 }

cmd/desync/location_test.go

@@ -12,6 +12,29 @@ func TestLocationEquality(t *testing.T) {
 	require.True(t, locationMatch("http://host/path", "http://host/path"))
 	require.True(t, locationMatch("http://host/path/", "http://host/path/"))
 	require.True(t, locationMatch("http://host/path", "http://host/path/"))
+	require.True(t, locationMatch("https://host/", "https://host"))
+	require.True(t, locationMatch("https://host", "https://host/"))
+	require.True(t, locationMatch("https://host", "https://host"))
+	require.True(t, locationMatch("https://host/", "https://host/"))
+	require.True(t, locationMatch("s3+https://example.com", "s3+https://example.com"))
+
+	// Equal URLs with globs
+	require.True(t, locationMatch("https://host/path*", "https://host/path"))
+	require.True(t, locationMatch("https://host/path*", "https://host/path/"))
+	require.True(t, locationMatch("https://*", "https://example.com"))
+	require.True(t, locationMatch("https://example.com/path/*", "https://example.com/path/another"))
+	require.True(t, locationMatch("https://example.com/path/*", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.com/*/*/", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.com/*/", "https://example.com/2022.01/"))
+	require.True(t, locationMatch("https://*/*/*", "https://example.com/path/another/"))
+	require.True(t, locationMatch("https://example.*", "https://example.com"))
+	require.True(t, locationMatch("*://example.com", "https://example.com"))
+	require.True(t, locationMatch("http*://example.com", "https://example.com"))
+	require.True(t, locationMatch("http*://example.com", "http://example.com"))
+	require.True(t, locationMatch("https://exampl?.*", "https://example.com"))
+	require.True(t, locationMatch("http://examp??.com", "http://example.com"))
+	require.True(t, locationMatch("https://example.com/?", "https://example.com/a"))
+	require.True(t, locationMatch("https://example.com/fo[a-z]", "https://example.com/foo"))
 
 	// Not equal URLs
 	require.False(t, locationMatch("http://host:8080/path", "http://host/path"))
@@ -20,10 +43,23 @@ func TestLocationEquality(t *testing.T) {
 	require.False(t, locationMatch("http://host1/path", "http://host2/path"))
 	require.False(t, locationMatch("sftp://host/path", "http://host/path"))
 	require.False(t, locationMatch("ssh://host/path", "/path"))
+	require.False(t, locationMatch("ssh://host/path", "/host/path"))
+	require.False(t, locationMatch("ssh://host/path", "/ssh/host/path"))
+
+	// Not equal URLs with globs
+	require.False(t, locationMatch("*", "https://example.com/path"))
+	require.False(t, locationMatch("https://*", "https://example.com/path"))
+	require.False(t, locationMatch("https://example.com/*", "https://example.com/path/another"))
+	require.False(t, locationMatch("https://example.com/path/*", "https://example.com/path"))
+	require.False(t, locationMatch("http://*", "https://example.com"))
+	require.False(t, locationMatch("http?://example.com", "http://example.com"))
+	require.False(t, locationMatch("https://example.com/123?", "https://example.com/12345"))
+	require.False(t, locationMatch("*://example.com", "https://example.com/123"))
 
 	// Equal paths
 	require.True(t, locationMatch("/path", "/path/../path"))
 	require.True(t, locationMatch("//path", "//path"))
+	require.True(t, locationMatch("//path", "/path"))
 	require.True(t, locationMatch("./path", "./path"))
 	require.True(t, locationMatch("path", "path/"))
 	require.True(t, locationMatch("path/..", "."))
@@ -32,11 +68,48 @@ func TestLocationEquality(t *testing.T) {
 		require.True(t, locationMatch("/path/to/somewhere", "\\path\\to\\somewhere\\"))
 	}
 
+	// Equal paths with globs
+	require.True(t, locationMatch("/path*", "/path/../path"))
+	require.True(t, locationMatch("/path*", "/path_1"))
+	require.True(t, locationMatch("/path/*", "/path/to"))
+	require.True(t, locationMatch("/path/*", "/path/to/"))
+	require.True(t, locationMatch("/path/*/", "/path/to/"))
+	require.True(t, locationMatch("/path/*/", "/path/to"))
+	require.True(t, locationMatch("/path/to/../*", "/path/another"))
+	require.True(t, locationMatch("/*", "/path"))
+	require.True(t, locationMatch("*", "path"))
+	require.True(t, locationMatch("/pat?", "/path"))
+	require.True(t, locationMatch("/pat?/?", "/path/1"))
+	require.True(t, locationMatch("path/*", "path/to"))
+	require.True(t, locationMatch("path/?", "path/1"))
+	require.True(t, locationMatch("?", "a"))
+	if runtime.GOOS == "windows" {
+		require.True(t, locationMatch("c:\\path\\to\\*", "c:\\path\\to\\somewhere\\"))
+		require.True(t, locationMatch("/path/to/*", "\\path\\to\\here\\"))
+		require.True(t, locationMatch("c:\\path\\to\\?", "c:\\path\\to\\1\\"))
+		require.True(t, locationMatch("/path/to/?", "\\path\\to\\1\\"))
+	}
+
 	// Not equal paths
 	require.False(t, locationMatch("/path", "path"))
 	require.False(t, locationMatch("/path/to", "path/to"))
 	require.False(t, locationMatch("/path/to", "/path/to/.."))
 	if runtime.GOOS == "windows" {
 		require.False(t, locationMatch("c:\\path1", "c:\\path2"))
 	}
+
+	// Not equal paths with globs
+	require.False(t, locationMatch("/path*", "/dir"))
+	require.False(t, locationMatch("/path*", "path"))
+	require.False(t, locationMatch("/path*", "/path/to"))
+	require.False(t, locationMatch("/path/*", "/path"))
+	require.False(t, locationMatch("/path/to/../*", "/path/to/another"))
+	require.False(t, locationMatch("/pat?", "/pat"))
+	require.False(t, locationMatch("/pat?", "/dir"))
+	if runtime.GOOS == "windows" {
+		require.True(t, locationMatch("c:\\path\\to\\*", "c:\\path\\to\\"))
+		require.True(t, locationMatch("/path/to/*", "\\path\\to\\"))
+		require.True(t, locationMatch("c:\\path\\to\\?", "c:\\path\\to\\123\\"))
+		require.True(t, locationMatch("/path/to/?", "\\path\\to\\123\\"))
+	}
 }

cmd/desync/pull.go

@@ -36,7 +36,10 @@ func runPull(ctx context.Context, opt pullOptions, args []string) error {
 	// SSH only supports serving compressed chunks currently. And we really
 	// don't want to have to decompress every chunk to verify its checksum.
 	// Clients will do that anyway, so disable verification here.
-	sOpt := cfg.GetStoreOptionsFor(storeLocation)
+	sOpt, err := cfg.GetStoreOptionsFor(storeLocation)
+	if err != nil {
+		return err
+	}
 	sOpt.SkipVerify = true
 
 	// Open the local store to serve chunks from

cmd/desync/store.go

@@ -104,7 +104,11 @@ func storeFromLocation(location string, cmdOpt cmdStoreOptions) (desync.Store, e
 
 	// Get any store options from the config if present and overwrite with settings from
 	// the command line
-	opt := cmdOpt.MergedWith(cfg.GetStoreOptionsFor(location))
+	configOptions, err := cfg.GetStoreOptionsFor(location)
+	if err != nil {
+		return nil, err
+	}
+	opt := cmdOpt.MergedWith(configOptions)
 
 	var s desync.Store
 	switch loc.Scheme {
@@ -231,7 +235,12 @@ func indexStoreFromLocation(location string, cmdOpt cmdStoreOptions) (desync.Ind
 	case strings.Contains(location, "\\"):
 		base = location[:strings.LastIndex(location, "\\")]
 	}
-	opt := cmdOpt.MergedWith(cfg.GetStoreOptionsFor(base))
+
+	configOptions, err := cfg.GetStoreOptionsFor(base)
+	if err != nil {
+		return nil, "", err
+	}
+	opt := cmdOpt.MergedWith(configOptions)
 
 	var s desync.IndexStore
 	switch loc.Scheme {

cmd/desync/verify.go

@@ -41,7 +41,11 @@ func runVerify(ctx context.Context, opt verifyOptions, args []string) error {
 	if opt.store == "" {
 		return errors.New("no store provided")
 	}
-	s, err := desync.NewLocalStore(opt.store, cfg.GetStoreOptionsFor(opt.store))
+	options, err := cfg.GetStoreOptionsFor(opt.store)
+	if err != nil {
+		return err
+	}
+	s, err := desync.NewLocalStore(opt.store, options)
 	if err != nil {
 		return err
 	}