Merge pull request #2424 from calmh/dbinstance

We should pass around db.Instance instead of leveldb.DB

cmd/syncthing/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"crypto/tls"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -42,9 +43,6 @@ import (
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
 
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/thejerf/suture"
 )
 
@@ -371,7 +369,7 @@ func main() {
 
 		if doUpgrade {
 			// Use leveldb database locks to protect against concurrent upgrades
-			_, err = leveldb.OpenFile(locations[locDatabase], &opt.Options{OpenFilesCacheCapacity: 100})
+			_, err = db.Open(locations[locDatabase])
 			if err != nil {
 				l.Infoln("Attempting upgrade through running Syncthing...")
 				err = upgradeViaRest()
@@ -617,21 +615,7 @@ func syncthingMain() {
 	}
 
 	dbFile := locations[locDatabase]
-	dbOpts := dbOpts(cfg)
+	ldb, err := db.Open(dbFile)
-	ldb, err := leveldb.OpenFile(dbFile, dbOpts)
-	if leveldbIsCorrupted(err) {
-		ldb, err = leveldb.RecoverFile(dbFile, dbOpts)
-	}
-	if leveldbIsCorrupted(err) {
-		// The database is corrupted, and we've tried to recover it but it
-		// didn't work. At this point there isn't much to do beyond dropping
-		// the database and reindexing...
-		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
-		if err := resetDB(); err != nil {
-			l.Fatalln("Remove database:", err)
-		}
-		ldb, err = leveldb.OpenFile(dbFile, dbOpts)
-	}
 	if err != nil {
 		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
 	}
@@ -642,7 +626,7 @@ func syncthingMain() {
 
 	// Remove database entries for folders that no longer exist in the config
 	folders := cfg.Folders()
-	for _, folder := range db.ListFolders(ldb) {
+	for _, folder := range ldb.ListFolders() {
 		if _, ok := folders[folder]; !ok {
 			l.Infof("Cleaning data for dropped folder %q", folder)
 			db.DropFolder(ldb, folder)
@@ -881,40 +865,6 @@ func loadConfig(cfgFile string) (*config.Wrapper, string, error) {
 	return cfg, myName, nil
 }
 
-func dbOpts(cfg *config.Wrapper) *opt.Options {
-	// Calculate a suitable database block cache capacity.
-
-	// Default is 8 MiB.
-	blockCacheCapacity := 8 << 20
-	// Increase block cache up to this maximum:
-	const maxCapacity = 64 << 20
-	// ... which we reach when the box has this much RAM:
-	const maxAtRAM = 8 << 30
-
-	if v := cfg.Options().DatabaseBlockCacheMiB; v != 0 {
-		// Use the value from the config, if it's set.
-		blockCacheCapacity = v << 20
-	} else if bytes, err := memorySize(); err == nil {
-		// We start at the default of 8 MiB and use larger values for machines
-		// with more memory.
-
-		if bytes > maxAtRAM {
-			// Cap the cache at maxCapacity when we reach maxAtRam amount of memory
-			blockCacheCapacity = maxCapacity
-		} else if bytes > maxAtRAM/maxCapacity*int64(blockCacheCapacity) {
-			// Grow from the default to maxCapacity at maxAtRam amount of memory
-			blockCacheCapacity = int(bytes * maxCapacity / maxAtRAM)
-		}
-		l.Infoln("Database block cache capacity", blockCacheCapacity/1024, "KiB")
-	}
-
-	return &opt.Options{
-		OpenFilesCacheCapacity: 100,
-		BlockCacheCapacity:     blockCacheCapacity,
-		WriteBuffer:            4 << 20,
-	}
-}
-
 func startAuditing(mainSvc *suture.Supervisor) {
 	auditFile := timestampedLoc(locAuditLog)
 	fd, err := os.OpenFile(auditFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
@@ -1166,19 +1116,3 @@ func checkShortIDs(cfg *config.Wrapper) error {
 	}
 	return nil
 }
-
-// A "better" version of leveldb's errors.IsCorrupted.
-func leveldbIsCorrupted(err error) bool {
-	switch {
-	case err == nil:
-		return false
-
-	case errors.IsCorrupted(err):
-		return true
-
-	case strings.Contains(err.Error(), "corrupted"):
-		return true
-	}
-
-	return false
-}

cmd/syncthing/main_test.go

@@ -14,9 +14,6 @@ import (
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 func TestFolderErrors(t *testing.T) {
@@ -38,7 +35,7 @@ func TestFolderErrors(t *testing.T) {
 		}
 	}
 
-	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
 
 	// Case 1 - new folder, directory and marker created
 

lib/config/config_test.go

@@ -56,7 +56,6 @@ func TestDefaultValues(t *testing.T) {
 		ProgressUpdateIntervalS: 5,
 		SymlinksEnabled:         true,
 		LimitBandwidthInLan:     false,
-		DatabaseBlockCacheMiB:   0,
 		MinHomeDiskFreePct:      1,
 		URURL:                   "https://data.syncthing.net/newdata",
 		URInitialDelayS:         1800,
@@ -180,7 +179,6 @@ func TestOverriddenValues(t *testing.T) {
 		ProgressUpdateIntervalS: 10,
 		SymlinksEnabled:         false,
 		LimitBandwidthInLan:     true,
-		DatabaseBlockCacheMiB:   42,
 		MinHomeDiskFreePct:      5.2,
 		URURL:                   "https://localhost/newdata",
 		URInitialDelayS:         800,

lib/config/optionsconfiguration.go

@@ -37,7 +37,6 @@ type OptionsConfiguration struct {
 	ProgressUpdateIntervalS int      `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
 	SymlinksEnabled         bool     `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
 	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
-	DatabaseBlockCacheMiB   int      `xml:"databaseBlockCacheMiB" json:"databaseBlockCacheMiB" default:"0"`
 	MinHomeDiskFreePct      float64  `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
 	ReleasesURL             string   `xml:"releasesURL" json:"releasesURL" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=30"`
 	AlwaysLocalNets         []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`

lib/db/blockmap.go

@@ -29,11 +29,11 @@ var blockFinder *BlockFinder
 const maxBatchSize = 256 << 10
 
 type BlockMap struct {
-	db     *leveldb.DB
+	db     *Instance
 	folder string
 }
 
-func NewBlockMap(db *leveldb.DB, folder string) *BlockMap {
+func NewBlockMap(db *Instance, folder string) *BlockMap {
 	return &BlockMap{
 		db:     db,
 		folder: folder,
@@ -146,10 +146,10 @@ func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
 }
 
 type BlockFinder struct {
-	db *leveldb.DB
+	db *Instance
 }
 
-func NewBlockFinder(db *leveldb.DB) *BlockFinder {
+func NewBlockFinder(db *Instance) *BlockFinder {
 	if blockFinder != nil {
 		return blockFinder
 	}

lib/db/blockmap_test.go

@@ -10,9 +10,6 @@ import (
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/protocol"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 func genBlocks(n int) []protocol.BlockInfo {
@@ -50,17 +47,14 @@ func init() {
 	}
 }
 
-func setup() (*leveldb.DB, *BlockFinder) {
+func setup() (*Instance, *BlockFinder) {
 	// Setup
 
-	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	db := OpenMemory()
-	if err != nil {
-		panic(err)
-	}
 	return db, NewBlockFinder(db)
 }
 
-func dbEmpty(db *leveldb.DB) bool {
+func dbEmpty(db *Instance) bool {
 	iter := db.NewIterator(nil, nil)
 	defer iter.Release()
 	if iter.Next() {

lib/db/leveldb_dbinstance.go

@@ -8,27 +8,64 @@ package db
 
 import (
 	"bytes"
+	"os"
 	"sort"
+	"strings"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64
 
-type dbInstance struct {
+type Instance struct {
 	*leveldb.DB
 }
 
-func newDBInstance(db *leveldb.DB) *dbInstance {
+func Open(file string) (*Instance, error) {
-	return &dbInstance{
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: 100,
+		WriteBuffer:            4 << 20,
+	}
+
+	db, err := leveldb.OpenFile(file, opts)
+	if leveldbIsCorrupted(err) {
+		db, err = leveldb.RecoverFile(file, opts)
+	}
+	if leveldbIsCorrupted(err) {
+		// The database is corrupted, and we've tried to recover it but it
+		// didn't work. At this point there isn't much to do beyond dropping
+		// the database and reindexing...
+		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
+		if err := os.RemoveAll(file); err != nil {
+			return nil, err
+		}
+		db, err = leveldb.OpenFile(file, opts)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return newDBInstance(db), nil
+}
+
+func OpenMemory() *Instance {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	return newDBInstance(db)
+}
+
+func newDBInstance(db *leveldb.DB) *Instance {
+	return &Instance{
 		DB: db,
 	}
 }
 
-func (db *dbInstance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
+func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
 	sort.Sort(fileList(fs)) // sort list on name, same as in the database
 
 	start := db.deviceKey(folder, device, nil)                            // before all folder/device files
@@ -126,7 +163,7 @@ func (db *dbInstance) genericReplace(folder, device []byte, fs []protocol.FileIn
 	return maxLocalVer
 }
 
-func (db *dbInstance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
+func (db *Instance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	// TODO: Return the remaining maxLocalVer?
 	return db.genericReplace(folder, device, fs, localSize, globalSize, func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		// Database has a file that we are missing. Remove it.
@@ -137,7 +174,7 @@ func (db *dbInstance) replace(folder, device []byte, fs []protocol.FileInfo, loc
 	})
 }
 
-func (db *dbInstance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
+func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -195,7 +232,7 @@ func (db *dbInstance) updateFiles(folder, device []byte, fs []protocol.FileInfo,
 	return maxLocalVer
 }
 
-func (db *dbInstance) withHave(folder, device []byte, truncate bool, fn Iterator) {
+func (db *Instance) withHave(folder, device []byte, truncate bool, fn Iterator) {
 	start := db.deviceKey(folder, device, nil)                            // before all folder/device files
 	limit := db.deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 
@@ -216,7 +253,7 @@ func (db *dbInstance) withHave(folder, device []byte, truncate bool, fn Iterator
 	}
 }
 
-func (db *dbInstance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
+func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
 	start := db.deviceKey(folder, nil, nil)                                                  // before all folder/device files
 	limit := db.deviceKey(folder, protocol.LocalDeviceID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
 
@@ -249,11 +286,11 @@ func (db *dbInstance) withAllFolderTruncated(folder []byte, fn func(device []byt
 	}
 }
 
-func (db *dbInstance) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
+func (db *Instance) getFile(folder, device, file []byte) (protocol.FileInfo, bool) {
 	return getFile(db, db.deviceKey(folder, device, file))
 }
 
-func (db *dbInstance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
+func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
 	k := db.globalKey(folder, file)
 
 	t := db.newReadOnlyTransaction()
@@ -290,7 +327,7 @@ func (db *dbInstance) getGlobal(folder, file []byte, truncate bool) (FileIntf, b
 	return fi, true
 }
 
-func (db *dbInstance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
+func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -333,7 +370,7 @@ func (db *dbInstance) withGlobal(folder, prefix []byte, truncate bool, fn Iterat
 	}
 }
 
-func (db *dbInstance) availability(folder, file []byte) []protocol.DeviceID {
+func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	k := db.globalKey(folder, file)
 	bs, err := db.Get(k, nil)
 	if err == leveldb.ErrNotFound {
@@ -361,7 +398,7 @@ func (db *dbInstance) availability(folder, file []byte) []protocol.DeviceID {
 	return devices
 }
 
-func (db *dbInstance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
+func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
 	start := db.globalKey(folder, nil)
 	limit := db.globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff})
 
@@ -452,7 +489,7 @@ nextFile:
 	}
 }
 
-func (db *dbInstance) listFolders() []string {
+func (db *Instance) ListFolders() []string {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -476,7 +513,7 @@ func (db *dbInstance) listFolders() []string {
 	return folders
 }
 
-func (db *dbInstance) dropFolder(folder []byte) {
+func (db *Instance) dropFolder(folder []byte) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -501,7 +538,7 @@ func (db *dbInstance) dropFolder(folder []byte) {
 	dbi.Release()
 }
 
-func (db *dbInstance) checkGlobals(folder []byte, globalSize *sizeTracker) {
+func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -560,11 +597,11 @@ func (db *dbInstance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 //	   folder (64 bytes)
 //	   device (32 bytes)
 //	   name (variable size)
-func (db *dbInstance) deviceKey(folder, device, file []byte) []byte {
+func (db *Instance) deviceKey(folder, device, file []byte) []byte {
 	return db.deviceKeyInto(nil, folder, device, file)
 }
 
-func (db *dbInstance) deviceKeyInto(k []byte, folder, device, file []byte) []byte {
+func (db *Instance) deviceKeyInto(k []byte, folder, device, file []byte) []byte {
 	reqLen := 1 + 64 + 32 + len(file)
 	if len(k) < reqLen {
 		k = make([]byte, reqLen)
@@ -579,11 +616,11 @@ func (db *dbInstance) deviceKeyInto(k []byte, folder, device, file []byte) []byt
 	return k[:reqLen]
 }
 
-func (db *dbInstance) deviceKeyName(key []byte) []byte {
+func (db *Instance) deviceKeyName(key []byte) []byte {
 	return key[1+64+32:]
 }
 
-func (db *dbInstance) deviceKeyFolder(key []byte) []byte {
+func (db *Instance) deviceKeyFolder(key []byte) []byte {
 	folder := key[1 : 1+64]
 	izero := bytes.IndexByte(folder, 0)
 	if izero < 0 {
@@ -592,7 +629,7 @@ func (db *dbInstance) deviceKeyFolder(key []byte) []byte {
 	return folder[:izero]
 }
 
-func (db *dbInstance) deviceKeyDevice(key []byte) []byte {
+func (db *Instance) deviceKeyDevice(key []byte) []byte {
 	return key[1+64 : 1+64+32]
 }
 
@@ -600,7 +637,7 @@ func (db *dbInstance) deviceKeyDevice(key []byte) []byte {
 //	   keyTypeGlobal (1 byte)
 //	   folder (64 bytes)
 //	   name (variable size)
-func (db *dbInstance) globalKey(folder, file []byte) []byte {
+func (db *Instance) globalKey(folder, file []byte) []byte {
 	k := make([]byte, 1+64+len(file))
 	k[0] = KeyTypeGlobal
 	if len(folder) > 64 {
@@ -611,11 +648,11 @@ func (db *dbInstance) globalKey(folder, file []byte) []byte {
 	return k
 }
 
-func (db *dbInstance) globalKeyName(key []byte) []byte {
+func (db *Instance) globalKeyName(key []byte) []byte {
 	return key[1+64:]
 }
 
-func (db *dbInstance) globalKeyFolder(key []byte) []byte {
+func (db *Instance) globalKeyFolder(key []byte) []byte {
 	folder := key[1 : 1+64]
 	izero := bytes.IndexByte(folder, 0)
 	if izero < 0 {
@@ -635,3 +672,19 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	err := tf.UnmarshalXDR(bs)
 	return tf, err
 }
+
+// A "better" version of leveldb's errors.IsCorrupted.
+func leveldbIsCorrupted(err error) bool {
+	switch {
+	case err == nil:
+		return false
+
+	case errors.IsCorrupted(err):
+		return true
+
+	case strings.Contains(err.Error(), "corrupted"):
+		return true
+	}
+
+	return false
+}

lib/db/leveldb_test.go

@@ -16,7 +16,7 @@ func TestDeviceKey(t *testing.T) {
 	dev := []byte("device67890123456789012345678901")
 	name := []byte("name")
 
-	db := &dbInstance{}
+	db := &Instance{}
 
 	key := db.deviceKey(fld, dev, name)
 
@@ -38,7 +38,7 @@ func TestGlobalKey(t *testing.T) {
 	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
 	name := []byte("name")
 
-	db := &dbInstance{}
+	db := &Instance{}
 
 	key := db.globalKey(fld, name)
 

lib/db/leveldb_transactions.go

@@ -16,10 +16,10 @@ import (
 // A readOnlyTransaction represents a database snapshot.
 type readOnlyTransaction struct {
 	*leveldb.Snapshot
-	db *dbInstance
+	db *Instance
 }
 
-func (db *dbInstance) newReadOnlyTransaction() readOnlyTransaction {
+func (db *Instance) newReadOnlyTransaction() readOnlyTransaction {
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -46,7 +46,7 @@ type readWriteTransaction struct {
 	*leveldb.Batch
 }
 
-func (db *dbInstance) newReadWriteTransaction() readWriteTransaction {
+func (db *Instance) newReadWriteTransaction() readWriteTransaction {
 	t := db.newReadOnlyTransaction()
 	return readWriteTransaction{
 		readOnlyTransaction: t,

lib/db/namespaced.go

@@ -17,13 +17,13 @@ import (
 // NamespacedKV is a simple key-value store using a specific namespace within
 // a leveldb.
 type NamespacedKV struct {
-	db     *leveldb.DB
+	db     *Instance
 	prefix []byte
 }
 
 // NewNamespacedKV returns a new NamespacedKV that lives in the namespace
 // specified by the prefix.
-func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV {
+func NewNamespacedKV(db *Instance, prefix string) *NamespacedKV {
 	return &NamespacedKV{
 		db:     db,
 		prefix: []byte(prefix),

lib/db/namespaced_test.go

@@ -9,16 +9,10 @@ package db
 import (
 	"testing"
 	"time"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 func TestNamespacedInt(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	n1 := NewNamespacedKV(ldb, "foo")
 	n2 := NewNamespacedKV(ldb, "bar")
@@ -53,10 +47,7 @@ func TestNamespacedInt(t *testing.T) {
 }
 
 func TestNamespacedTime(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	n1 := NewNamespacedKV(ldb, "foo")
 
@@ -73,10 +64,7 @@ func TestNamespacedTime(t *testing.T) {
 }
 
 func TestNamespacedString(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	n1 := NewNamespacedKV(ldb, "foo")
 
@@ -92,10 +80,7 @@ func TestNamespacedString(t *testing.T) {
 }
 
 func TestNamespacedReset(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	n1 := NewNamespacedKV(ldb, "foo")
 

lib/db/set.go

@@ -18,14 +18,13 @@ import (
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type FileSet struct {
 	localVersion map[protocol.DeviceID]int64
 	mutex        sync.Mutex
 	folder       string
-	db           *dbInstance
+	db           *Instance
 	blockmap     *BlockMap
 	localSize    sizeTracker
 	globalSize   sizeTracker
@@ -93,11 +92,11 @@ func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
 	return s.files, s.deleted, s.bytes
 }
 
-func NewFileSet(folder string, db *leveldb.DB) *FileSet {
+func NewFileSet(folder string, db *Instance) *FileSet {
 	var s = FileSet{
 		localVersion: make(map[protocol.DeviceID]int64),
 		folder:       folder,
-		db:           newDBInstance(db),
+		db:           db,
 		blockmap:     NewBlockMap(db, folder),
 		mutex:        sync.NewMutex(),
 	}
@@ -239,17 +238,10 @@ func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
 	return s.globalSize.Size()
 }
 
-// ListFolders returns the folder IDs seen in the database.
-func ListFolders(db *leveldb.DB) []string {
-	i := newDBInstance(db)
-	return i.listFolders()
-}
-
 // DropFolder clears out all information related to the given folder from the
 // database.
-func DropFolder(db *leveldb.DB, folder string) {
+func DropFolder(db *Instance, folder string) {
-	i := newDBInstance(db)
+	db.dropFolder([]byte(folder))
-	i.dropFolder([]byte(folder))
 	bm := &BlockMap{
 		db:     db,
 		folder: folder,

lib/db/set_test.go

@@ -15,8 +15,6 @@ import (
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 var remoteDevice0, remoteDevice1 protocol.DeviceID
@@ -96,11 +94,7 @@ func (l fileList) String() string {
 }
 
 func TestGlobalSet(t *testing.T) {
-
+	ldb := db.OpenMemory()
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	m := db.NewFileSet("test", ldb)
 
@@ -303,10 +297,7 @@ func TestGlobalSet(t *testing.T) {
 }
 
 func TestNeedWithInvalid(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s := db.NewFileSet("test", ldb)
 
@@ -343,10 +334,7 @@ func TestNeedWithInvalid(t *testing.T) {
 }
 
 func TestUpdateToInvalid(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s := db.NewFileSet("test", ldb)
 
@@ -378,10 +366,7 @@ func TestUpdateToInvalid(t *testing.T) {
 }
 
 func TestInvalidAvailability(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s := db.NewFileSet("test", ldb)
 
@@ -419,10 +404,7 @@ func TestInvalidAvailability(t *testing.T) {
 }
 
 func TestGlobalReset(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	m := db.NewFileSet("test", ldb)
 
@@ -460,10 +442,7 @@ func TestGlobalReset(t *testing.T) {
 }
 
 func TestNeed(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	m := db.NewFileSet("test", ldb)
 
@@ -501,10 +480,7 @@ func TestNeed(t *testing.T) {
 }
 
 func TestLocalVersion(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	m := db.NewFileSet("test", ldb)
 
@@ -534,10 +510,7 @@ func TestLocalVersion(t *testing.T) {
 }
 
 func TestListDropFolder(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s0 := db.NewFileSet("test0", ldb)
 	local1 := []protocol.FileInfo{
@@ -558,7 +531,7 @@ func TestListDropFolder(t *testing.T) {
 	// Check that we have both folders and their data is in the global list
 
 	expectedFolderList := []string{"test0", "test1"}
-	if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
+	if actualFolderList := ldb.ListFolders(); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
 		t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList)
 	}
 	if l := len(globalList(s0)); l != 3 {
@@ -573,7 +546,7 @@ func TestListDropFolder(t *testing.T) {
 	db.DropFolder(ldb, "test1")
 
 	expectedFolderList = []string{"test0"}
-	if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
+	if actualFolderList := ldb.ListFolders(); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
 		t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList)
 	}
 	if l := len(globalList(s0)); l != 3 {
@@ -585,10 +558,7 @@ func TestListDropFolder(t *testing.T) {
 }
 
 func TestGlobalNeedWithInvalid(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s := db.NewFileSet("test1", ldb)
 
@@ -625,10 +595,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 }
 
 func TestLongPath(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	s := db.NewFileSet("test", ldb)
 

lib/db/virtualmtime.go

@@ -9,8 +9,6 @@ package db
 import (
 	"fmt"
 	"time"
-
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 // This type encapsulates a repository of mtimes for platforms where file mtimes
@@ -25,7 +23,7 @@ type VirtualMtimeRepo struct {
 	ns *NamespacedKV
 }
 
-func NewVirtualMtimeRepo(ldb *leveldb.DB, folder string) *VirtualMtimeRepo {
+func NewVirtualMtimeRepo(ldb *Instance, folder string) *VirtualMtimeRepo {
 	prefix := string(KeyTypeVirtualMtime) + folder
 
 	return &VirtualMtimeRepo{

lib/db/virtualmtime_test.go

@@ -9,16 +9,10 @@ package db
 import (
 	"testing"
 	"time"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 func TestVirtualMtimeRepo(t *testing.T) {
-	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := OpenMemory()
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// A few repos so we can ensure they don't pollute each other
 	repo1 := NewVirtualMtimeRepo(ldb, "folder1")

lib/model/model.go

@@ -33,7 +33,6 @@ import (
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/versioner"
-	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/thejerf/suture"
 )
 
@@ -64,7 +63,7 @@ type Model struct {
 	*suture.Supervisor
 
 	cfg               *config.Wrapper
-	db                *leveldb.DB
+	db                *db.Instance
 	finder            *db.BlockFinder
 	progressEmitter   *ProgressEmitter
 	id                protocol.DeviceID
@@ -99,7 +98,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB, protectedFiles []string) *Model {
+func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *db.Instance, protectedFiles []string) *Model {
 	m := &Model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {

lib/model/model_test.go

@@ -22,8 +22,6 @@ import (
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 var device1, device2 protocol.DeviceID
@@ -90,7 +88,7 @@ func init() {
 }
 
 func TestRequest(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
@@ -167,7 +165,7 @@ func BenchmarkIndex_100(b *testing.B) {
 }
 
 func benchmarkIndex(b *testing.B, nfiles int) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
@@ -196,7 +194,7 @@ func BenchmarkIndexUpdate_10000_1(b *testing.B) {
 }
 
 func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
@@ -261,7 +259,7 @@ func (FakeConnection) Statistics() protocol.Statistics {
 }
 
 func BenchmarkRequest(b *testing.B) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
@@ -317,7 +315,7 @@ func TestDeviceRename(t *testing.T) {
 	}
 	cfg := config.Wrap("tmpconfig.xml", rawCfg)
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
 	fc := FakeConnection{
@@ -391,7 +389,7 @@ func TestClusterConfig(t *testing.T) {
 		},
 	}
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 
 	m := NewModel(config.Wrap("/tmp/test", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(cfg.Folders[0])
@@ -463,7 +461,7 @@ func TestIgnores(t *testing.T) {
 	ioutil.WriteFile("testdata/.stfolder", nil, 0644)
 	ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.StartFolderRO("default")
@@ -538,7 +536,7 @@ func TestIgnores(t *testing.T) {
 }
 
 func TestRefuseUnknownBits(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
@@ -576,7 +574,7 @@ func TestRefuseUnknownBits(t *testing.T) {
 }
 
 func TestROScanRecovery(t *testing.T) {
-	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
@@ -660,7 +658,7 @@ func TestROScanRecovery(t *testing.T) {
 }
 
 func TestRWScanRecovery(t *testing.T) {
-	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
@@ -744,7 +742,7 @@ func TestRWScanRecovery(t *testing.T) {
 }
 
 func TestGlobalDirectoryTree(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
@@ -994,7 +992,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 }
 
 func TestGlobalDirectorySelfFixing(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
@@ -1168,7 +1166,7 @@ func BenchmarkTree_100_10(b *testing.B) {
 }
 
 func benchmarkTree(b *testing.B, n1, n2 int) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	m.ServeBackground()
@@ -1186,7 +1184,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 }
 
 func TestIgnoreDelete(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 
 	// This folder should ignore external deletes

lib/model/rwfolder_test.go

@@ -12,12 +12,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/sync"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 func init() {
@@ -69,7 +67,7 @@ func TestHandleFile(t *testing.T) {
 	requiredFile := existingFile
 	requiredFile.Blocks = blocks[1:]
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
@@ -125,7 +123,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	requiredFile := existingFile
 	requiredFile.Blocks = blocks[1:]
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
@@ -187,7 +185,7 @@ func TestCopierFinder(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 	requiredFile.Name = "file2"
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 	// Update index
@@ -264,7 +262,7 @@ func TestCopierCleanup(t *testing.T) {
 		return true
 	}
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
@@ -313,7 +311,7 @@ func TestCopierCleanup(t *testing.T) {
 // Make sure that the copier routine hashes the content when asked, and pulls
 // if it fails to find the block.
 func TestLastResortPulling(t *testing.T) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 
@@ -387,7 +385,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 	}
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
@@ -480,7 +478,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	}
 	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
 
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	db := db.OpenMemory()
 	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
 	m.AddFolder(defaultFolderConfig)
 

lib/stats/device.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type DeviceStatistics struct {
@@ -22,7 +21,7 @@ type DeviceStatisticsReference struct {
 	device string
 }
 
-func NewDeviceStatisticsReference(ldb *leveldb.DB, device string) *DeviceStatisticsReference {
+func NewDeviceStatisticsReference(ldb *db.Instance, device string) *DeviceStatisticsReference {
 	prefix := string(db.KeyTypeDeviceStatistic) + device
 	return &DeviceStatisticsReference{
 		ns:     db.NewNamespacedKV(ldb, prefix),

lib/stats/folder.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type FolderStatistics struct {
@@ -28,7 +27,7 @@ type LastFile struct {
 	Deleted  bool      `json:"deleted"`
 }
 
-func NewFolderStatisticsReference(ldb *leveldb.DB, folder string) *FolderStatisticsReference {
+func NewFolderStatisticsReference(ldb *db.Instance, folder string) *FolderStatisticsReference {
 	prefix := string(db.KeyTypeFolderStatistic) + folder
 	return &FolderStatisticsReference{
 		ns:     db.NewNamespacedKV(ldb, prefix),