swarm/shed: initial implementation

swarm/shed/internal/db.go

@@ -0,0 +1,84 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const openFileLimit = 128
+
+type DB struct {
+	ldb *leveldb.DB
+}
+
+func NewDB(path string) (db *DB, err error) {
+	ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit})
+	if err != nil {
+		return nil, err
+	}
+	db = &DB{ldb: ldb}
+
+	if _, err = db.getSchema(); err != nil {
+		if err == leveldb.ErrNotFound {
+			if err = db.putSchema(schema{
+				Fields:  make(map[string]fieldSpec),
+				Indexes: make(map[byte]indexSpec),
+			}); err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+	return db, nil
+}
+
+func (db *DB) Put(key []byte, value []byte) (err error) {
+	metrics.GetOrRegisterCounter("DB.put", nil).Inc(1)
+
+	return db.ldb.Put(key, value, nil)
+}
+
+func (db *DB) Get(key []byte) (value []byte, err error) {
+	metrics.GetOrRegisterCounter("DB.get", nil).Inc(1)
+
+	return db.ldb.Get(key, nil)
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.ldb.Delete(key, nil)
+}
+
+func (db *DB) NewIterator() iterator.Iterator {
+	metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1)
+
+	return db.ldb.NewIterator(nil, nil)
+}
+
+func (db *DB) WriteBatch(batch *leveldb.Batch) error {
+	metrics.GetOrRegisterCounter("DB.write", nil).Inc(1)
+
+	return db.ldb.Write(batch, nil)
+}
+
+func (db *DB) Close() (err error) {
+	return db.ldb.Close()
+}

swarm/shed/internal/db_test.go

@@ -0,0 +1,103 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestNewDB(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	s, err := db.getSchema()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Fields == nil {
+		t.Error("schema fields are empty")
+	}
+	if len(s.Fields) != 0 {
+		t.Errorf("got schema fields length %v, want %v", len(s.Fields), 0)
+	}
+	if s.Indexes == nil {
+		t.Error("schema indexes are empty")
+	}
+	if len(s.Indexes) != 0 {
+		t.Errorf("got schema indexes length %v, want %v", len(s.Indexes), 0)
+	}
+}
+
+func TestDB_persistence(t *testing.T) {
+	dir, err := ioutil.TempDir("", "shed-test-persistence")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	db, err := NewDB(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stringField, err := db.NewStringField("preserve-me")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "persistent value"
+	err = stringField.Put(want)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db2, err := NewDB(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stringField2, err := db2.NewStringField("preserve-me")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := stringField2.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got string %q, want %q", got, want)
+	}
+}
+
+func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) {
+	t.Helper()
+
+	dir, err := ioutil.TempDir("", "shed-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cleanupFunc = func() { os.RemoveAll(dir) }
+	db, err = NewDB(dir)
+	if err != nil {
+		cleanupFunc()
+		t.Fatal(err)
+	}
+	return db, cleanupFunc
+}

swarm/shed/internal/field_json.go

@@ -0,0 +1,68 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"encoding/json"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type JSONField struct {
+	db  *DB
+	key []byte
+}
+
+func (db *DB) NewJSONField(name string) (f JSONField, err error) {
+	key, err := db.schemaFieldKey(name, "json")
+	if err != nil {
+		return f, err
+	}
+	return JSONField{
+		db:  db,
+		key: key,
+	}, nil
+}
+
+func (f JSONField) Unmarshal(val interface{}) (err error) {
+	b, err := f.db.Get(f.key)
+	if err != nil {
+		// Q: should we ignore not found
+		// if err == leveldb.ErrNotFound {
+		// 	return nil
+		// }
+		return err
+	}
+	return json.Unmarshal(b, val)
+}
+
+func (f JSONField) Put(val interface{}) (err error) {
+	b, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	return f.db.Put(f.key, b)
+}
+
+func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) {
+	b, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	batch.Put(f.key, b)
+	return nil
+}

swarm/shed/internal/field_json_test.go

@@ -0,0 +1,125 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func TestJSONField(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	complexField, err := db.NewJSONField("complex-field")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	type complexStructure struct {
+		A string
+	}
+
+	t.Run("unmarshal empty", func(t *testing.T) {
+		var s complexStructure
+		err := complexField.Unmarshal(&s)
+		if err != leveldb.ErrNotFound {
+			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+		}
+		want := ""
+		if s.A != want {
+			t.Errorf("got string %q, want %q", s.A, want)
+		}
+	})
+
+	t.Run("put", func(t *testing.T) {
+		want := complexStructure{
+			A: "simple string value",
+		}
+		err = complexField.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got complexStructure
+		err = complexField.Unmarshal(&got)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got.A != want.A {
+			t.Errorf("got string %q, want %q", got.A, want.A)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			want := complexStructure{
+				A: "overwritten string value",
+			}
+			err = complexField.Put(want)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got complexStructure
+			err = complexField.Unmarshal(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.A != want.A {
+				t.Errorf("got string %q, want %q", got.A, want.A)
+			}
+		})
+	})
+
+	t.Run("put in batch", func(t *testing.T) {
+		batch := new(leveldb.Batch)
+		want := complexStructure{
+			A: "simple string batch value",
+		}
+		complexField.PutInBatch(batch, want)
+		err = db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got complexStructure
+		err := complexField.Unmarshal(&got)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got.A != want.A {
+			t.Errorf("got string %q, want %q", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			batch := new(leveldb.Batch)
+			want := complexStructure{
+				A: "overwritten string batch value",
+			}
+			complexField.PutInBatch(batch, want)
+			err = db.WriteBatch(batch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got complexStructure
+			err := complexField.Unmarshal(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.A != want.A {
+				t.Errorf("got string %q, want %q", got, want)
+			}
+		})
+	})
+}