Add HIBP tests

action/hibp.go

@@ -73,7 +73,7 @@ func (s *Action) hibpDump(ctx context.Context, force bool, dumps []string) error
 		return err
 	}
 
-	scanner, err := hibpdump.New(dumps)
+	scanner, err := hibpdump.New(dumps...)
 	if err != nil {
 		return exitError(ctx, ExitUsage, err, "Failed to create new HIBP Dump scanner: %s", err)
 	}

utils/hibp/api/client.go

@@ -23,8 +23,9 @@ func Lookup(ctx context.Context, shaSum string) (uint64, error) {
 		return 0, errors.Errorf("invalid shasum")
 	}
 
-	prefix := strings.ToUpper(shaSum[:5])
-	suffix := strings.ToUpper(shaSum[5:])
+	shaSum = strings.ToUpper(shaSum)
+	prefix := shaSum[:5]
+	suffix := shaSum[5:]
 
 	var count uint64
 	url := fmt.Sprintf("%s/range/%s", URL, prefix)

utils/hibp/api/client_test.go

@@ -0,0 +1,58 @@
+package api
+
+import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookup(t *testing.T) {
+	ctx := context.Background()
+
+	match := "match"
+	noMatch := "no match"
+	matchSum := sha1sum(match)
+	var matchCount uint64 = 324567
+
+	reqCnt := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		reqCnt++
+		if reqCnt < 2 {
+			http.Error(w, "fake error", http.StatusInternalServerError)
+			return
+		}
+		if strings.TrimPrefix(r.URL.String(), "/range/") == matchSum[:5] {
+			fmt.Fprintf(w, matchSum[5:10]+":1\n")                            // invalid
+			fmt.Fprintf(w, matchSum[5:39]+":3234879\n")                      // invalid
+			fmt.Fprintf(w, matchSum[5:]+":\n")                               // invalid
+			fmt.Fprintf(w, matchSum[5:]+"\n")                                // invalid
+			fmt.Fprintf(w, fmt.Sprintf("%s:%d\n", matchSum[5:], matchCount)) // valid
+			return
+		}
+		http.Error(w, "not found", http.StatusNotFound)
+	}))
+	defer ts.Close()
+	URL = ts.URL
+
+	// test with one entry
+	count, err := Lookup(ctx, matchSum)
+	assert.NoError(t, err)
+	assert.Equal(t, matchCount, count)
+
+	// add another one
+	count, err = Lookup(ctx, sha1sum(noMatch))
+	assert.NoError(t, err)
+	assert.Equal(t, uint64(0), count)
+}
+
+func sha1sum(data string) string {
+	h := sha1.New()
+	_, _ = h.Write([]byte(data))
+	return fmt.Sprintf("%X", h.Sum(nil))
+}

utils/hibp/dump/scanner.go

@@ -21,7 +21,7 @@ type Scanner struct {
 }
 
 // New creates a new scanner
-func New(dumps []string) (*Scanner, error) {
+func New(dumps ...string) (*Scanner, error) {
 	ok := make([]string, 0, len(dumps))
 	for _, dump := range dumps {
 		if !fsutil.IsFile(dump) {
@@ -114,6 +114,9 @@ func isSorted(fn string) bool {
 		}
 
 		line := scanner.Text()
+		if len(line) > 40 {
+			line = line[:40]
+		}
 		if line < lastLine {
 			return false
 		}

utils/hibp/dump/scanner_test.go

@@ -0,0 +1,72 @@
+package dump
+
+import (
+	"compress/gzip"
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const testHibpSample = `000000005AD76BD555C1D6D771DE417A4B87E4B4
+00000000A8DAE4228F821FB418F59826079BF368:42
+00000000DD7F2A1C68A35673713783CA390C9E93:42
+00000001E225B908BAC31C56DB04D892E47536E0:42
+00000008CD1806EB7B9B46A8F87690B2AC16F617:42
+0000000A0E3B9F25FF41DE4B5AC238C2D545C7A8:42
+0000000A1D4B746FAA3FD526FF6D5BC8052FDB38:42
+0000000CAEF405439D57847A8657218C618160B2:42
+0000000FC1C08E6454BED24F463EA2129E254D43:42
+00000010F4B38525354491E099EB1796278544B1`
+
+func TestScanner(t *testing.T) {
+	td, err := ioutil.TempDir("", "gopass-")
+	assert.NoError(t, err)
+
+	defer func() {
+		_ = os.RemoveAll(td)
+	}()
+
+	ctx := context.Background()
+
+	// no hibp dump, no scanner
+	_, err = New()
+	assert.Error(t, err)
+
+	// setup file and env
+	fn := filepath.Join(td, "dump.txt")
+	assert.NoError(t, ioutil.WriteFile(fn, []byte(testHibpSample), 0644))
+
+	scanner, err := New(fn)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, scanner.LookupBatch(ctx, []string{"foobar"}))
+
+	// gzip
+	fn = filepath.Join(td, "dump.txt.gz")
+	assert.NoError(t, testWriteGZ(fn, []byte(testHibpSample)))
+
+	scanner, err = New(fn)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, scanner.LookupBatch(ctx, []string{"foobar"}))
+}
+
+func testWriteGZ(fn string, buf []byte) error {
+	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = fh.Close()
+	}()
+
+	gzw := gzip.NewWriter(fh)
+	defer func() {
+		_ = gzw.Close()
+	}()
+
+	_, err = gzw.Write(buf)
+	return err
+}