Merge pull request #1275 from calmh/kv-cache

Namespaced key value store and value cache

internal/db/blockmap.go

@@ -199,7 +199,7 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b
 //	   file name (variable size)
 func toBlockKey(hash []byte, folder, file string) []byte {
 	o := make([]byte, 1+64+32+len(file))
-	o[0] = keyTypeBlock
+	o[0] = KeyTypeBlock
 	copy(o[1:], []byte(folder))
 	copy(o[1+64:], []byte(hash))
 	copy(o[1+64+32:], []byte(file))
@@ -210,7 +210,7 @@ func fromBlockKey(data []byte) (string, string) {
 	if len(data) < 1+64+32+1 {
 		panic("Incorrect key length")
 	}
-	if data[0] != keyTypeBlock {
+	if data[0] != KeyTypeBlock {
 		panic("Incorrect key type")
 	}
 

internal/db/leveldb.go

@@ -50,9 +50,11 @@ func clock(v int64) int64 {
 }
 
 const (
-	keyTypeDevice = iota
+	KeyTypeDevice = iota
-	keyTypeGlobal
+	KeyTypeGlobal
-	keyTypeBlock
+	KeyTypeBlock
+	KeyTypeDeviceStatistic
+	KeyTypeFolderStatistic
 )
 
 type fileVersion struct {
@@ -112,7 +114,7 @@ const batchFlushSize = 64
 //	   name (variable size)
 func deviceKey(folder, device, file []byte) []byte {
 	k := make([]byte, 1+64+32+len(file))
-	k[0] = keyTypeDevice
+	k[0] = KeyTypeDevice
 	if len(folder) > 64 {
 		panic("folder name too long")
 	}
@@ -145,7 +147,7 @@ func deviceKeyDevice(key []byte) []byte {
 //	   name (variable size)
 func globalKey(folder, file []byte) []byte {
 	k := make([]byte, 1+64+len(file))
-	k[0] = keyTypeGlobal
+	k[0] = KeyTypeGlobal
 	if len(folder) > 64 {
 		panic("folder name too long")
 	}
@@ -901,8 +903,6 @@ outer:
 func ldbListFolders(db *leveldb.DB) []string {
 	runtime.GC()
 
-	start := []byte{keyTypeGlobal}
-	limit := []byte{keyTypeGlobal + 1}
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -917,7 +917,7 @@ func ldbListFolders(db *leveldb.DB) []string {
 		snap.Release()
 	}()
 
-	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
 	defer dbi.Release()
 
 	folderExists := make(map[string]bool)
@@ -955,9 +955,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
 	}()
 
 	// Remove all items related to the given folder from the device->file bucket
-	start := []byte{keyTypeDevice}
+	dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
-	limit := []byte{keyTypeDevice + 1}
-	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
 	for dbi.Next() {
 		itemFolder := deviceKeyFolder(dbi.Key())
 		if bytes.Compare(folder, itemFolder) == 0 {
@@ -967,9 +965,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
 	dbi.Release()
 
 	// Remove all items related to the given folder from the global bucket
-	start = []byte{keyTypeGlobal}
+	dbi = snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
-	limit = []byte{keyTypeGlobal + 1}
-	dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
 	for dbi.Next() {
 		itemFolder := globalKeyFolder(dbi.Key())
 		if bytes.Compare(folder, itemFolder) == 0 {

internal/db/namespaced.go

@@ -0,0 +1,106 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// NamespacedKV is a simple key-value store using a specific namespace within
+// a leveldb.
+type NamespacedKV struct {
+	db     *leveldb.DB
+	prefix []byte
+}
+
+// NewNamespacedKV returns a new NamespacedKV that lives in the namespace
+// specified by the prefix.
+func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV {
+	return &NamespacedKV{
+		db:     db,
+		prefix: []byte(prefix),
+	}
+}
+
+// PutInt64 stores a new int64. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutInt64(key string, val int64) {
+	keyBs := append(n.prefix, []byte(key)...)
+	var valBs [8]byte
+	binary.BigEndian.PutUint64(valBs[:], uint64(val))
+	n.db.Put(keyBs, valBs[:], nil)
+}
+
+// Int64 returns the stored value interpreted as an int64 and a boolean that
+// is false if no value was stored at the key.
+func (n *NamespacedKV) Int64(key string) (int64, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return 0, false
+	}
+	val := binary.BigEndian.Uint64(valBs)
+	return int64(val), true
+}
+
+// PutTime stores a new time.Time. Any existing value (even if of another
+// type) is overwritten.
+func (n *NamespacedKV) PutTime(key string, val time.Time) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, _ := val.MarshalBinary() // never returns an error
+	n.db.Put(keyBs, valBs, nil)
+}
+
+// Time returns the stored value interpreted as a time.Time and a boolean
+// that is false if no value was stored at the key.
+func (n NamespacedKV) Time(key string) (time.Time, bool) {
+	var t time.Time
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return t, false
+	}
+	err = t.UnmarshalBinary(valBs)
+	return t, err == nil
+}
+
+// PutString stores a new string. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutString(key, val string) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Put(keyBs, []byte(val), nil)
+}
+
+// String returns the stored value interpreted as a string and a boolean that
+// is false if no value was stored at the key.
+func (n NamespacedKV) String(key string) (string, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return "", false
+	}
+	return string(valBs), true
+}
+
+// Delete deletes the specified key. It is allowed to delete a nonexistent
+// key.
+func (n NamespacedKV) Delete(key string) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Delete(keyBs, nil)
+}

internal/db/namespaced_test.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestNamespacedInt(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+	n2 := NewNamespacedKV(ldb, "bar")
+
+	// Key is missing to start with
+
+	if v, ok := n1.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+
+	n1.PutInt64("test", 42)
+
+	// It should now exist in n1
+
+	if v, ok := n1.Int64("test"); v != 42 || !ok {
+		t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok)
+	}
+
+	// ... but not in n2, which is in a different namespace
+
+	if v, ok := n2.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+
+	n1.Delete("test")
+
+	// It should no longer exist
+
+	if v, ok := n1.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+}
+
+func TestNamespacedTime(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+
+	if v, ok := n1.Time("test"); v != (time.Time{}) || ok {
+		t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok)
+	}
+
+	now := time.Now()
+	n1.PutTime("test", now)
+
+	if v, ok := n1.Time("test"); v != now || !ok {
+		t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok)
+	}
+}
+
+func TestNamespacedString(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+
+	if v, ok := n1.String("test"); v != "" || ok {
+		t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+	}
+
+	n1.PutString("test", "yo")
+
+	if v, ok := n1.String("test"); v != "yo" || !ok {
+		t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok)
+	}
+}

internal/stats/device.go

@@ -19,91 +19,45 @@ import (
 	"time"
 
 	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-const (
-	deviceStatisticTypeLastSeen = iota
-)
-
-var deviceStatisticsTypes = []byte{
-	deviceStatisticTypeLastSeen,
-}
-
 type DeviceStatistics struct {
 	LastSeen time.Time
 }
 
 type DeviceStatisticsReference struct {
-	db     *leveldb.DB
+	ns     *db.NamespacedKV
 	device protocol.DeviceID
 }
 
-func NewDeviceStatisticsReference(db *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference {
+func NewDeviceStatisticsReference(ldb *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference {
+	prefix := string(db.KeyTypeDeviceStatistic) + device.String()
 	return &DeviceStatisticsReference{
-		db:     db,
+		ns:     db.NewNamespacedKV(ldb, prefix),
 		device: device,
 	}
 }
 
-func (s *DeviceStatisticsReference) key(stat byte) []byte {
-	k := make([]byte, 1+1+32)
-	k[0] = keyTypeDeviceStatistic
-	k[1] = stat
-	copy(k[1+1:], s.device[:])
-	return k
-}
-
 func (s *DeviceStatisticsReference) GetLastSeen() time.Time {
-	value, err := s.db.Get(s.key(deviceStatisticTypeLastSeen), nil)
+	t, ok := s.ns.Time("lastSeen")
-	if err != nil {
+	if !ok {
-		if err != leveldb.ErrNotFound {
+		// The default here is 1970-01-01 as opposed to the default
-			l.Warnln("DeviceStatisticsReference: Failed loading last seen value for", s.device, ":", err)
+		// time.Time{} from s.ns
-		}
-		return time.Unix(0, 0)
-	}
-
-	rtime := time.Time{}
-	err = rtime.UnmarshalBinary(value)
-	if err != nil {
-		l.Warnln("DeviceStatisticsReference: Failed parsing last seen value for", s.device, ":", err)
 		return time.Unix(0, 0)
 	}
 	if debug {
-		l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, rtime)
+		l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, t)
 	}
-	return rtime
+	return t
 }
 
 func (s *DeviceStatisticsReference) WasSeen() {
 	if debug {
 		l.Debugln("stats.DeviceStatisticsReference.WasSeen:", s.device)
 	}
-	value, err := time.Now().MarshalBinary()
+	s.ns.PutTime("lastSeen", time.Now())
-	if err != nil {
-		l.Warnln("DeviceStatisticsReference: Failed serializing last seen value for", s.device, ":", err)
-		return
-	}
-
-	err = s.db.Put(s.key(deviceStatisticTypeLastSeen), value, nil)
-	if err != nil {
-		l.Warnln("Failed serializing last seen value for", s.device, ":", err)
-	}
-}
-
-// Never called, maybe because it's worth while to keep the data
-// or maybe because we have no easy way of knowing that a device has been removed.
-func (s *DeviceStatisticsReference) Delete() error {
-	for _, stype := range deviceStatisticsTypes {
-		err := s.db.Delete(s.key(stype), nil)
-		if debug && err == nil {
-			l.Debugln("stats.DeviceStatisticsReference.Delete:", s.device, stype)
-		}
-		if err != nil && err != leveldb.ErrNotFound {
-			return err
-		}
-	}
-	return nil
 }
 
 func (s *DeviceStatisticsReference) GetStatistics() DeviceStatistics {

internal/stats/folder.go

@@ -16,118 +16,59 @@
 package stats
 
 import (
-	"encoding/binary"
 	"time"
 
+	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-const (
-	folderStatisticTypeLastFile = iota
-)
-
-var folderStatisticsTypes = []byte{
-	folderStatisticTypeLastFile,
-}
-
 type FolderStatistics struct {
-	LastFile *LastFile
+	LastFile LastFile
 }
 
 type FolderStatisticsReference struct {
-	db     *leveldb.DB
+	ns     *db.NamespacedKV
 	folder string
 }
 
-func NewFolderStatisticsReference(db *leveldb.DB, folder string) *FolderStatisticsReference {
-	return &FolderStatisticsReference{
-		db:     db,
-		folder: folder,
-	}
-}
-
-func (s *FolderStatisticsReference) key(stat byte) []byte {
-	k := make([]byte, 1+1+64)
-	k[0] = keyTypeFolderStatistic
-	k[1] = stat
-	copy(k[1+1:], s.folder[:])
-	return k
-}
-
-func (s *FolderStatisticsReference) GetLastFile() *LastFile {
-	value, err := s.db.Get(s.key(folderStatisticTypeLastFile), nil)
-	if err != nil {
-		if err != leveldb.ErrNotFound {
-			l.Warnln("FolderStatisticsReference: Failed loading last file filename value for", s.folder, ":", err)
-		}
-		return nil
-	}
-
-	file := LastFile{}
-	err = file.UnmarshalBinary(value)
-	if err != nil {
-		l.Warnln("FolderStatisticsReference: Failed loading last file value for", s.folder, ":", err)
-		return nil
-	}
-	return &file
-}
-
-func (s *FolderStatisticsReference) ReceivedFile(filename string) {
-	f := LastFile{
-		Filename: filename,
-		At:       time.Now(),
-	}
-	if debug {
-		l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder)
-	}
-
-	value, err := f.MarshalBinary()
-	if err != nil {
-		l.Warnln("FolderStatisticsReference: Failed serializing last file value for", s.folder, ":", err)
-		return
-	}
-
-	err = s.db.Put(s.key(folderStatisticTypeLastFile), value, nil)
-	if err != nil {
-		l.Warnln("Failed update last file value for", s.folder, ":", err)
-	}
-}
-
-// Never called, maybe because it's worth while to keep the data
-// or maybe because we have no easy way of knowing that a folder has been removed.
-func (s *FolderStatisticsReference) Delete() error {
-	for _, stype := range folderStatisticsTypes {
-		err := s.db.Delete(s.key(stype), nil)
-		if debug && err == nil {
-			l.Debugln("stats.FolderStatisticsReference.Delete:", s.folder, stype)
-		}
-		if err != nil && err != leveldb.ErrNotFound {
-			return err
-		}
-	}
-	return nil
-}
-
-func (s *FolderStatisticsReference) GetStatistics() FolderStatistics {
-	return FolderStatistics{
-		LastFile: s.GetLastFile(),
-	}
-}
-
 type LastFile struct {
 	At       time.Time
 	Filename string
 }
 
-func (f *LastFile) MarshalBinary() ([]byte, error) {
+func NewFolderStatisticsReference(ldb *leveldb.DB, folder string) *FolderStatisticsReference {
-	buf := make([]byte, 8+len(f.Filename))
+	prefix := string(db.KeyTypeFolderStatistic) + folder
-	binary.BigEndian.PutUint64(buf[:8], uint64(f.At.Unix()))
+	return &FolderStatisticsReference{
-	copy(buf[8:], []byte(f.Filename))
+		ns:     db.NewNamespacedKV(ldb, prefix),
-	return buf, nil
+		folder: folder,
+	}
 }
 
-func (f *LastFile) UnmarshalBinary(buf []byte) error {
+func (s *FolderStatisticsReference) GetLastFile() LastFile {
-	f.At = time.Unix(int64(binary.BigEndian.Uint64(buf[:8])), 0)
+	at, ok := s.ns.Time("lastFileAt")
-	f.Filename = string(buf[8:])
+	if !ok {
-	return nil
+		return LastFile{}
+	}
+	file, ok := s.ns.String("lastFileName")
+	if !ok {
+		return LastFile{}
+	}
+	return LastFile{
+		At:       at,
+		Filename: file,
+	}
+}
+
+func (s *FolderStatisticsReference) ReceivedFile(filename string) {
+	if debug {
+		l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder, filename)
+	}
+	s.ns.PutTime("lastFileAt", time.Now())
+	s.ns.PutString("lastFileName", filename)
+}
+
+func (s *FolderStatisticsReference) GetStatistics() FolderStatistics {
+	return FolderStatistics{
+		LastFile: s.GetLastFile(),
+	}
 }

internal/stats/leveldb.go

@@ -1,22 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package stats
-
-// Same key space as files/leveldb.go keyType* constants
-const (
-	keyTypeDeviceStatistic = iota + 30
-	keyTypeFolderStatistic
-)