lib/db: Refactor: use a Lowlevel type underneath Instance (ref #5198) (#5212)

This adds a thin type that holds the state associated with the
leveldb.DB, leaving the huge Instance type more or less stateless. Also
moves some keying stuff into the DB package so that other packages need
not know the keying specifics.

(This does not, yet, fix the cmd/stindex program, in order to keep the
diff size down. Hence the keying constants are still exported.)

cmd/stindex/dump.go

@@ -16,7 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Instance) {
+func dump(ldb *db.Lowlevel) {
 	it := ldb.NewIterator(nil, nil)
 	for it.Next() {
 		key := it.Key()

cmd/stindex/dumpsize.go

@@ -37,7 +37,7 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Instance) {
+func dumpsize(ldb *db.Lowlevel) {
 	h := &ElementHeap{}
 	heap.Init(h)
 

cmd/syncthing/main.go

@@ -712,11 +712,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	if err != nil {
 		l.Fatalln("Error opening database:", err)
 	}
+	if err := db.UpdateSchema(ldb); err != nil {
+		l.Fatalln("Database schema:", err)
+	}
 
 	if runtimeOptions.resetDeltaIdxs {
 		l.Infoln("Reinitializing delta index IDs")
-		ldb.DropLocalDeltaIndexIDs()
+		db.DropDeltaIndexIDs(ldb)
-		ldb.DropRemoteDeltaIndexIDs()
 	}
 
 	protectedFiles := []string{
@@ -737,7 +739,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// Grab the previously running version string from the database.
 
-	miscDB := db.NewNamespacedKV(ldb, string(db.KeyTypeMiscData))
+	miscDB := db.NewMiscDataNamespace(ldb)
 	prevVersion, _ := miscDB.String("prevVersion")
 
 	// Strip away prerelease/beta stuff and just compare the release
@@ -753,7 +755,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 		// Drop delta indexes in case we've changed random stuff we
 		// shouldn't have. We will resend our index on next connect.
-		ldb.DropLocalDeltaIndexIDs()
+		db.DropDeltaIndexIDs(ldb)
 
 		// Remember the new version.
 		miscDB.PutString("prevVersion", Version)

lib/db/benchmark_test.go

@@ -45,7 +45,7 @@ func lazyInitBenchFileSet() {
 	replace(benchS, protocol.LocalDeviceID, firstHalf)
 }
 
-func tempDB() (*db.Instance, string) {
+func tempDB() (*db.Lowlevel, string) {
 	dir, err := ioutil.TempDir("", "syncthing")
 	if err != nil {
 		panic(err)

lib/db/blockmap.go

@@ -22,14 +22,14 @@ var blockFinder *BlockFinder
 const maxBatchSize = 1000
 
 type BlockMap struct {
-	db     *Instance
+	db     *Lowlevel
 	folder uint32
 }
 
-func NewBlockMap(db *Instance, folder uint32) *BlockMap {
+func NewBlockMap(db *Lowlevel, folder string) *BlockMap {
 	return &BlockMap{
 		db:     db,
-		folder: folder,
+		folder: db.folderIdx.ID([]byte(folder)),
 	}
 }
 
@@ -139,10 +139,10 @@ func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
 }
 
 type BlockFinder struct {
-	db *Instance
+	db *Lowlevel
 }
 
-func NewBlockFinder(db *Instance) *BlockFinder {
+func NewBlockFinder(db *Lowlevel) *BlockFinder {
 	if blockFinder != nil {
 		return blockFinder
 	}

lib/db/blockmap_test.go

@@ -48,14 +48,14 @@ func init() {
 	}
 }
 
-func setup() (*Instance, *BlockFinder) {
+func setup() (*Lowlevel, *BlockFinder) {
 	// Setup
 
 	db := OpenMemory()
 	return db, NewBlockFinder(db)
 }
 
-func dbEmpty(db *Instance) bool {
+func dbEmpty(db *Lowlevel) bool {
 	iter := db.NewIterator(util.BytesPrefix([]byte{KeyTypeBlock}), nil)
 	defer iter.Release()
 	return !iter.Next()
@@ -68,7 +68,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal("db not empty")
 	}
 
-	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
+	m := NewBlockMap(db, "folder1")
 
 	f3.Type = protocol.FileInfoTypeDirectory
 
@@ -152,8 +152,8 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 func TestBlockFinderLookup(t *testing.T) {
 	db, f := setup()
 
-	m1 := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
+	m1 := NewBlockMap(db, "folder1")
-	m2 := NewBlockMap(db, db.folderIdx.ID([]byte("folder2")))
+	m2 := NewBlockMap(db, "folder2")
 
 	err := m1.Add([]protocol.FileInfo{f1})
 	if err != nil {

lib/db/keyer_test.go

@@ -16,7 +16,7 @@ func TestDeviceKey(t *testing.T) {
 	dev := []byte("device67890123456789012345678901")
 	name := []byte("name")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
 
@@ -44,7 +44,7 @@ func TestGlobalKey(t *testing.T) {
 	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
 	name := []byte("name")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
 
@@ -69,7 +69,7 @@ func TestGlobalKey(t *testing.T) {
 func TestSequenceKey(t *testing.T) {
 	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	const seq = 1234567890
 	key := db.keyer.GenerateSequenceKey(nil, fld, seq)

lib/db/leveldb.go

@@ -31,7 +31,7 @@ func (vl VersionList) String() string {
 // update brings the VersionList up to date with file. It returns the updated
 // VersionList, a potentially removed old FileVersion and its index, as well as
 // the index where the new FileVersion was inserted.
-func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
+func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
 	removedAt, insertedAt = -1, -1
 	for i, v := range vl.Versions {
 		if bytes.Equal(v.Device, device) {

lib/db/leveldb_dbinstance.go

@@ -10,87 +10,28 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"os"
-	"sort"
-	"strings"
-	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator)
 
-type Instance struct {
+type instance struct {
-	committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
+	*Lowlevel
-	*leveldb.DB
-	location  string
-	folderIdx *smallIndex
-	deviceIdx *smallIndex
 	keyer keyer
 }
 
-func Open(file string) (*Instance, error) {
+func newInstance(ll *Lowlevel) *instance {
-	opts := &opt.Options{
+	return &instance{
-		OpenFilesCacheCapacity: 100,
+		Lowlevel: ll,
-		WriteBuffer:            4 << 20,
+		keyer:    newDefaultKeyer(ll.folderIdx, ll.deviceIdx),
 	}
-
-	db, err := leveldb.OpenFile(file, opts)
-	if leveldbIsCorrupted(err) {
-		db, err = leveldb.RecoverFile(file, opts)
-	}
-	if leveldbIsCorrupted(err) {
-		// The database is corrupted, and we've tried to recover it but it
-		// didn't work. At this point there isn't much to do beyond dropping
-		// the database and reindexing...
-		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
-		if err := os.RemoveAll(file); err != nil {
-			return nil, errorSuggestion{err, "failed to delete corrupted database"}
-		}
-		db, err = leveldb.OpenFile(file, opts)
-	}
-	if err != nil {
-		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
-	}
-
-	return newDBInstance(db, file)
 }
 
-func OpenMemory() *Instance {
+func (db *instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	ldb, _ := newDBInstance(db, "<memory>")
-	return ldb
-}
-
-func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
-	i := &Instance{
-		DB:        db,
-		location:  location,
-		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
-		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
-	}
-	i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
-	err := i.updateSchema()
-	return i, err
-}
-
-// Committed returns the number of items committed to the database since startup
-func (db *Instance) Committed() int64 {
-	return atomic.LoadInt64(&db.committed)
-}
-
-// Location returns the filesystem path where the database is stored
-func (db *Instance) Location() string {
-	return db.location
-}
-
-func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -131,7 +72,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
 	}
 }
 
-func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
+func (db *instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -146,7 +87,7 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	}
 }
 
-func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
+func (db *instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -158,7 +99,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
 	}
 }
 
-func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
+func (db *instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
 	if len(prefix) > 0 {
 		unslashedPrefix := prefix
 		if bytes.HasSuffix(prefix, []byte{'/'}) {
@@ -199,7 +140,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 	}
 }
 
-func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
+func (db *instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -226,7 +167,7 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
 	}
 }
 
-func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
+func (db *instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -271,14 +212,14 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 	}
 }
 
-func (db *Instance) getFile(key []byte) (protocol.FileInfo, bool) {
+func (db *instance) getFile(key []byte) (protocol.FileInfo, bool) {
 	if f, ok := db.getFileTrunc(key, false); ok {
 		return f.(protocol.FileInfo), true
 	}
 	return protocol.FileInfo{}, false
 }
 
-func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
+func (db *instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
 	bs, err := db.Get(key, nil)
 	if err == leveldb.ErrNotFound {
 		return nil, false
@@ -296,7 +237,7 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
 	return f, true
 }
 
-func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
+func (db *instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -304,7 +245,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 	return f, ok
 }
 
-func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
+func (db *instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
 	gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
 
 	bs, err := t.Get(gk, nil)
@@ -325,7 +266,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
 	return gk, dk, nil, false
 }
 
-func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
+func (db *instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
 	if len(prefix) > 0 {
 		unslashedPrefix := prefix
 		if bytes.HasSuffix(prefix, []byte{'/'}) {
@@ -370,7 +311,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	}
 }
 
-func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
+func (db *instance) availability(folder, file []byte) []protocol.DeviceID {
 	k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
 	bs, err := db.Get(k, nil)
 	if err == leveldb.ErrNotFound {
@@ -401,7 +342,7 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	return devices
 }
 
-func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
+func (db *instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
 	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
 		db.withNeedLocal(folder, truncate, fn)
 		return
@@ -473,7 +414,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 	}
 }
 
-func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
+func (db *instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -495,31 +436,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	}
 }
 
-func (db *Instance) ListFolders() []string {
+func (db *instance) dropFolder(folder []byte) {
-	t := db.newReadOnlyTransaction()
-	defer t.close()
-
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
-	defer dbi.Release()
-
-	folderExists := make(map[string]bool)
-	for dbi.Next() {
-		folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
-		if ok && !folderExists[string(folder)] {
-			folderExists[string(folder)] = true
-		}
-	}
-
-	folders := make([]string, 0, len(folderExists))
-	for k := range folderExists {
-		folders = append(folders, k)
-	}
-
-	sort.Strings(folders)
-	return folders
-}
-
-func (db *Instance) dropFolder(folder []byte) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -537,7 +454,7 @@ func (db *Instance) dropFolder(folder []byte) {
 	}
 }
 
-func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
+func (db *instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -556,7 +473,7 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	}
 }
 
-func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
+func (db *instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -604,7 +521,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	l.Debugf("db check completed for %q", folder)
 }
 
-func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
+func (db *instance) getIndexID(device, folder []byte) protocol.IndexID {
 	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	cur, err := db.Get(key, nil)
 	if err != nil {
@@ -619,7 +536,7 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
 	return id
 }
 
-func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
+func (db *instance) setIndexID(device, folder []byte, id protocol.IndexID) {
 	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	bs, _ := id.Marshal() // marshalling can't fail
 	if err := db.Put(key, bs, nil); err != nil {
@@ -627,44 +544,15 @@ func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
 	}
 }
 
-// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
+func (db *instance) dropMtimes(folder []byte) {
-// the database. This will cause a full index transmission on the next
-// connection.
-func (db *Instance) DropLocalDeltaIndexIDs() {
-	db.dropDeltaIndexIDs(true)
-}
-
-// DropRemoteDeltaIndexIDs removes all index IDs for the other devices than
-// the local one from the database. This will cause them to send us a full
-// index on the next connection.
-func (db *Instance) DropRemoteDeltaIndexIDs() {
-	db.dropDeltaIndexIDs(false)
-}
-
-func (db *Instance) dropDeltaIndexIDs(local bool) {
-	t := db.newReadWriteTransaction()
-	defer t.close()
-
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
-	defer dbi.Release()
-
-	for dbi.Next() {
-		device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
-		if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
-			t.Delete(dbi.Key())
-		}
-	}
-}
-
-func (db *Instance) dropMtimes(folder []byte) {
 	db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
 }
 
-func (db *Instance) dropFolderMeta(folder []byte) {
+func (db *instance) dropFolderMeta(folder []byte) {
 	db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
 }
 
-func (db *Instance) dropPrefix(prefix []byte) {
+func (db *instance) dropPrefix(prefix []byte) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -701,22 +589,6 @@ func unmarshalVersionList(data []byte) (VersionList, bool) {
 	return vl, true
 }
 
-// A "better" version of leveldb's errors.IsCorrupted.
-func leveldbIsCorrupted(err error) bool {
-	switch {
-	case err == nil:
-		return false
-
-	case errors.IsCorrupted(err):
-		return true
-
-	case strings.Contains(err.Error(), "corrupted"):
-		return true
-	}
-
-	return false
-}
-
 type errorSuggestion struct {
 	inner      error
 	suggestion string

lib/db/leveldb_dbinstance_updateschema.go

@@ -38,8 +38,17 @@ func (e databaseDowngradeError) Error() string {
 	return fmt.Sprintf("Syncthing %s required", e.minSyncthingVersion)
 }
 
-func (db *Instance) updateSchema() error {
+func UpdateSchema(ll *Lowlevel) error {
-	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
+	updater := &schemaUpdater{newInstance(ll)}
+	return updater.updateSchema()
+}
+
+type schemaUpdater struct {
+	*instance
+}
+
+func (db *schemaUpdater) updateSchema() error {
+	miscDB := NewMiscDataNamespace(db.Lowlevel)
 	prevVersion, _ := miscDB.Int64("dbVersion")
 
 	if prevVersion > dbVersion {
@@ -77,7 +86,7 @@ func (db *Instance) updateSchema() error {
 	return nil
 }
 
-func (db *Instance) updateSchema0to1() {
+func (db *schemaUpdater) updateSchema0to1() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -159,7 +168,7 @@ func (db *Instance) updateSchema0to1() {
 
 // updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
 // to allow iteration in sequence order (simplifies sending indexes).
-func (db *Instance) updateSchema1to2() {
+func (db *schemaUpdater) updateSchema1to2() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -178,7 +187,7 @@ func (db *Instance) updateSchema1to2() {
 }
 
 // updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
-func (db *Instance) updateSchema2to3() {
+func (db *schemaUpdater) updateSchema2to3() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -209,7 +218,7 @@ func (db *Instance) updateSchema2to3() {
 // release candidates (dbVersion 3 and 4)
 // https://github.com/syncthing/syncthing/issues/5007
 // https://github.com/syncthing/syncthing/issues/5053
-func (db *Instance) updateSchemaTo5() {
+func (db *schemaUpdater) updateSchemaTo5() {
 	t := db.newReadWriteTransaction()
 	var nk []byte
 	for _, folderStr := range db.ListFolders() {
@@ -221,7 +230,7 @@ func (db *Instance) updateSchemaTo5() {
 	db.updateSchema2to3()
 }
 
-func (db *Instance) updateSchema5to6() {
+func (db *schemaUpdater) updateSchema5to6() {
 	// For every local file with the Invalid bit set, clear the Invalid bit and
 	// set LocalFlags = FlagLocalIgnored.
 

lib/db/leveldb_test.go

@@ -7,107 +7,20 @@
 package db
 
 import (
-	"os"
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func TestDropIndexIDs(t *testing.T) {
-	db := OpenMemory()
-
-	d1 := []byte("device67890123456789012345678901")
-	d2 := []byte("device12345678901234567890123456")
-
-	// Set some index IDs
-
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
-	db.setIndexID(d1, []byte("foo"), 3)
-	db.setIndexID(d1, []byte("bar"), 4)
-	db.setIndexID(d2, []byte("foo"), 5)
-	db.setIndexID(d2, []byte("bar"), 6)
-
-	// Verify them
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 3 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 4 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 5 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 6 {
-		t.Fatal("fail remote 4")
-	}
-
-	// Drop the local ones, verify only they got dropped
-
-	db.DropLocalDeltaIndexIDs()
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 0 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 0 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 3 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 4 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 5 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 6 {
-		t.Fatal("fail remote 4")
-	}
-
-	// Set local ones again
-
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
-
-	// Drop the remote ones, verify only they got dropped
-
-	db.DropRemoteDeltaIndexIDs()
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 0 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 0 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 0 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 0 {
-		t.Fatal("fail remote 4")
-	}
-}
-
 func TestIgnoredFiles(t *testing.T) {
 	ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons")
 	if err != nil {
 		t.Fatal(err)
 	}
-	db, _ := newDBInstance(ldb, "<memory>")
+	db := NewLowlevel(ldb, "<memory>")
+	UpdateSchema(db)
+
 	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
 
 	// The contents of the database are like this:
@@ -228,11 +141,13 @@ func TestUpdate0to3(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	db, _ := newDBInstance(ldb, "<memory>")
+
+	db := newInstance(NewLowlevel(ldb, "<memory>"))
+	updater := schemaUpdater{db}
 
 	folder := []byte(update0to3Folder)
 
-	db.updateSchema0to1()
+	updater.updateSchema0to1()
 
 	if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
 		t.Error("File prefixed by '/' was not removed during transition to schema 1")
@@ -242,7 +157,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Invalid file wasn't added to global list")
 	}
 
-	db.updateSchema1to2()
+	updater.updateSchema1to2()
 
 	found := false
 	db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
@@ -263,7 +178,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Local file wasn't added to sequence bucket", err)
 	}
 
-	db.updateSchema2to3()
+	updater.updateSchema2to3()
 
 	need := map[string]protocol.FileInfo{
 		haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
@@ -288,22 +203,17 @@ func TestUpdate0to3(t *testing.T) {
 }
 
 func TestDowngrade(t *testing.T) {
-	loc := "testdata/downgrade.db"
+	db := OpenMemory()
-	db, err := Open(loc)
+	UpdateSchema(db) // sets the min version etc
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		db.Close()
-		os.RemoveAll(loc)
-	}()
 
-	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
+	// Bump the database version to something newer than we actually support
+	miscDB := NewMiscDataNamespace(db)
 	miscDB.PutInt64("dbVersion", dbVersion+1)
 	l.Infoln(dbVersion)
 
-	db.Close()
+	// Pretend we just opened the DB and attempt to update it again
-	db, err = Open(loc)
+	err := UpdateSchema(db)
+
 	if err, ok := err.(databaseDowngradeError); !ok {
 		t.Fatal("Expected error due to database downgrade, got", err)
 	} else if err.minSyncthingVersion != dbMinSyncthingVersion {

lib/db/leveldb_transactions.go

@@ -8,7 +8,6 @@ package db
 
 import (
 	"bytes"
-	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -18,10 +17,10 @@ import (
 // A readOnlyTransaction represents a database snapshot.
 type readOnlyTransaction struct {
 	*leveldb.Snapshot
-	db *Instance
+	db *instance
 }
 
-func (db *Instance) newReadOnlyTransaction() readOnlyTransaction {
+func (db *instance) newReadOnlyTransaction() readOnlyTransaction {
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -48,7 +47,7 @@ type readWriteTransaction struct {
 	*leveldb.Batch
 }
 
-func (db *Instance) newReadWriteTransaction() readWriteTransaction {
+func (db *instance) newReadWriteTransaction() readWriteTransaction {
 	t := db.newReadOnlyTransaction()
 	return readWriteTransaction{
 		readOnlyTransaction: t,
@@ -72,7 +71,6 @@ func (t readWriteTransaction) flush() {
 	if err := t.db.Write(t.Batch, nil); err != nil {
 		panic(err)
 	}
-	atomic.AddInt64(&t.db.committed, int64(t.Batch.Len()))
 }
 
 func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protocol.FileInfo) {

lib/db/lowlevel.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"os"
+	"strings"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+const (
+	dbMaxOpenFiles = 100
+	dbWriteBuffer  = 4 << 20
+)
+
+// Lowlevel is the lowest level database interface. It has a very simple
+// purpose: hold the actual *leveldb.DB database, and the in-memory state
+// that belong to that database. In the same way that a single on disk
+// database can only be opened once, there should be only one Lowlevel for
+// any given *leveldb.DB.
+type Lowlevel struct {
+	committed int64 // atomic, must come first
+	*leveldb.DB
+	location  string
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+}
+
+// Open attempts to open the database at the given location, and runs
+// recovery on it if opening fails. Worst case, if recovery is not possible,
+// the database is erased and created from scratch.
+func Open(location string) (*Lowlevel, error) {
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: dbMaxOpenFiles,
+		WriteBuffer:            dbWriteBuffer,
+	}
+
+	db, err := leveldb.OpenFile(location, opts)
+	if leveldbIsCorrupted(err) {
+		db, err = leveldb.RecoverFile(location, opts)
+	}
+	if leveldbIsCorrupted(err) {
+		// The database is corrupted, and we've tried to recover it but it
+		// didn't work. At this point there isn't much to do beyond dropping
+		// the database and reindexing...
+		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
+		if err := os.RemoveAll(location); err != nil {
+			return nil, errorSuggestion{err, "failed to delete corrupted database"}
+		}
+		db, err = leveldb.OpenFile(location, opts)
+	}
+	if err != nil {
+		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
+	}
+	return NewLowlevel(db, location), nil
+}
+
+// OpenMemory returns a new Lowlevel referencing an in-memory database.
+func OpenMemory() *Lowlevel {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	return NewLowlevel(db, "<memory>")
+}
+
+// Location returns the filesystem path where the database is stored
+func (db *Lowlevel) Location() string {
+	return db.location
+}
+
+// ListFolders returns the list of folders currently in the database
+func (db *Lowlevel) ListFolders() []string {
+	return db.folderIdx.Values()
+}
+
+// Committed returns the number of items committed to the database since startup
+func (db *Lowlevel) Committed() int64 {
+	return atomic.LoadInt64(&db.committed)
+}
+
+func (db *Lowlevel) Put(key, val []byte, wo *opt.WriteOptions) error {
+	atomic.AddInt64(&db.committed, 1)
+	return db.DB.Put(key, val, wo)
+}
+
+func (db *Lowlevel) Delete(key []byte, wo *opt.WriteOptions) error {
+	atomic.AddInt64(&db.committed, 1)
+	return db.DB.Delete(key, wo)
+}
+
+// NewLowlevel wraps the given *leveldb.DB into a *lowlevel
+func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
+	return &Lowlevel{
+		DB:        db,
+		location:  location,
+		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
+	}
+}
+
+// A "better" version of leveldb's errors.IsCorrupted.
+func leveldbIsCorrupted(err error) bool {
+	switch {
+	case err == nil:
+		return false
+
+	case errors.IsCorrupted(err):
+		return true
+
+	case strings.Contains(err.Error(), "corrupted"):
+		return true
+	}
+
+	return false
+}

lib/db/meta.go

@@ -20,6 +20,7 @@ type metadataTracker struct {
 	mut     sync.RWMutex
 	counts  CountsSet
 	indexes map[metaKey]int // device ID + local flags -> index in counts
+	dirty   bool
 }
 
 type metaKey struct {
@@ -55,18 +56,31 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
 
 // toDB saves the marshalled metadataTracker to the given db, under the key
 // corresponding to the given folder
-func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
+func (m *metadataTracker) toDB(db *instance, folder []byte) error {
 	key := db.keyer.GenerateFolderMetaKey(nil, folder)
+
+	m.mut.RLock()
+	defer m.mut.RUnlock()
+
+	if !m.dirty {
+		return nil
+	}
+
 	bs, err := m.Marshal()
 	if err != nil {
 		return err
 	}
-	return db.Put(key, bs, nil)
+	err = db.Put(key, bs, nil)
+	if err == nil {
+		m.dirty = false
+	}
+
+	return err
 }
 
 // fromDB initializes the metadataTracker from the marshalled data found in
 // the database under the key corresponding to the given folder
-func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
+func (m *metadataTracker) fromDB(db *instance, folder []byte) error {
 	key := db.keyer.GenerateFolderMetaKey(nil, folder)
 	bs, err := db.Get(key, nil)
 	if err != nil {
@@ -99,6 +113,7 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
 	}
 
 	m.mut.Lock()
+	m.dirty = true
 
 	if flags := f.FileLocalFlags(); flags == 0 {
 		// Account regular files in the zero-flags bucket.
@@ -141,6 +156,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
 	}
 
 	m.mut.Lock()
+	m.dirty = true
 
 	if flags := f.FileLocalFlags(); flags == 0 {
 		// Remove regular files from the zero-flags bucket
@@ -194,6 +210,7 @@ func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32,
 // resetAll resets all metadata for the given device
 func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
 	m.mut.Lock()
+	m.dirty = true
 	for i, c := range m.counts.Counts {
 		if bytes.Equal(c.DeviceID, dev[:]) {
 			m.counts.Counts[i] = Counts{
@@ -209,6 +226,7 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
 // sequence number
 func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
 	m.mut.Lock()
+	m.dirty = true
 
 	for i, c := range m.counts.Counts {
 		if bytes.Equal(c.DeviceID, dev[:]) {
@@ -285,6 +303,7 @@ func (m *metadataTracker) Created() time.Time {
 func (m *metadataTracker) SetCreated() {
 	m.mut.Lock()
 	m.counts.Created = time.Now().UnixNano()
+	m.dirty = true
 	m.mut.Unlock()
 }
 

lib/db/namespaced.go

@@ -17,13 +17,13 @@ import (
 // NamespacedKV is a simple key-value store using a specific namespace within
 // a leveldb.
 type NamespacedKV struct {
-	db     *Instance
+	db     *Lowlevel
 	prefix []byte
 }
 
 // NewNamespacedKV returns a new NamespacedKV that lives in the namespace
 // specified by the prefix.
-func NewNamespacedKV(db *Instance, prefix string) *NamespacedKV {
+func NewNamespacedKV(db *Lowlevel, prefix string) *NamespacedKV {
 	return &NamespacedKV{
 		db:     db,
 		prefix: []byte(prefix),
@@ -157,3 +157,23 @@ func (n NamespacedKV) Delete(key string) {
 	keyBs := append(n.prefix, []byte(key)...)
 	n.db.Delete(keyBs, nil)
 }
+
+// Well known namespaces that can be instantiated without knowing the key
+// details.
+
+// NewDeviceStatisticsNamespace creates a KV namespace for device statistics
+// for the given device.
+func NewDeviceStatisticsNamespace(db *Lowlevel, device string) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeDeviceStatistic)+device)
+}
+
+// NewFolderStatisticsNamespace creates a KV namespace for folder statistics
+// for the given folder.
+func NewFolderStatisticsNamespace(db *Lowlevel, folder string) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeFolderStatistic)+folder)
+}
+
+// NewMiscDateNamespace creates a KV namespace for miscellaneous metadata.
+func NewMiscDataNamespace(db *Lowlevel) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeMiscData))
+}

lib/db/set.go

@@ -21,12 +21,13 @@ import (
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type FileSet struct {
 	folder   string
 	fs       fs.Filesystem
-	db       *Instance
+	db       *instance
 	blockmap *BlockMap
 	meta     *metadataTracker
 
@@ -66,12 +67,14 @@ func init() {
 	}
 }
 
-func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet {
+func NewFileSet(folder string, fs fs.Filesystem, ll *Lowlevel) *FileSet {
+	db := newInstance(ll)
+
 	var s = FileSet{
 		folder:      folder,
 		fs:          fs,
 		db:          db,
-		blockmap:    NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
+		blockmap:    NewBlockMap(ll, folder),
 		meta:        newMetadataTracker(),
 		updateMutex: sync.NewMutex(),
 	}
@@ -310,7 +313,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
 
 func (s *FileSet) MtimeFS() *fs.MtimeFS {
 	prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
-	kv := NewNamespacedKV(s.db, string(prefix))
+	kv := NewNamespacedKV(s.db.Lowlevel, string(prefix))
 	return fs.NewMtimeFS(s.fs, kv)
 }
 
@@ -320,15 +323,26 @@ func (s *FileSet) ListDevices() []protocol.DeviceID {
 
 // DropFolder clears out all information related to the given folder from the
 // database.
-func DropFolder(db *Instance, folder string) {
+func DropFolder(ll *Lowlevel, folder string) {
+	db := newInstance(ll)
 	db.dropFolder([]byte(folder))
 	db.dropMtimes([]byte(folder))
 	db.dropFolderMeta([]byte(folder))
-	bm := &BlockMap{
+	bm := NewBlockMap(ll, folder)
-		db:     db,
-		folder: db.folderIdx.ID([]byte(folder)),
-	}
 	bm.Drop()
+
+	// Also clean out the folder ID mapping.
+	db.folderIdx.Delete([]byte(folder))
+}
+
+// DropDeltaIndexIDs removes all delta index IDs from the database.
+// This will cause a full index transmission on the next connection.
+func DropDeltaIndexIDs(db *Lowlevel) {
+	dbi := db.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
+	defer dbi.Release()
+	for dbi.Next() {
+		db.Delete(dbi.Key(), nil)
+	}
 }
 
 func normalizeFilenames(fs []protocol.FileInfo) {

lib/db/smallindex.go

@@ -8,6 +8,7 @@ package db
 
 import (
 	"encoding/binary"
+	"sort"
 
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -46,8 +47,11 @@ func (i *smallIndex) load() {
 	for it.Next() {
 		val := string(it.Value())
 		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
+		if val != "" {
+			// Empty value means the entry has been deleted.
 			i.id2val[id] = val
 			i.val2id[val] = id
+		}
 		if id >= i.nextID {
 			i.nextID = id + 1
 		}
@@ -96,3 +100,45 @@ func (i *smallIndex) Val(id uint32) ([]byte, bool) {
 
 	return []byte(val), true
 }
+
+func (i *smallIndex) Delete(val []byte) {
+	i.mut.Lock()
+	defer i.mut.Unlock()
+
+	// Check the reverse mapping to get the ID for the value.
+	if id, ok := i.val2id[string(val)]; ok {
+		// Generate the corresponding database key.
+		key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
+		copy(key, i.prefix)
+		binary.BigEndian.PutUint32(key[len(i.prefix):], id)
+
+		// Put an empty value into the database. This indicates that the
+		// entry does not exist any more and prevents the ID from being
+		// reused in the future.
+		i.db.Put(key, []byte{}, nil)
+
+		// Delete reverse mapping.
+		delete(i.id2val, id)
+	}
+
+	// Delete forward mapping.
+	delete(i.val2id, string(val))
+}
+
+// Values returns the set of values in the index
+func (i *smallIndex) Values() []string {
+	// In principle this method should return [][]byte because all the other
+	// methods deal in []byte keys. However, in practice, where it's used
+	// wants a []string and it's easier to just create that here rather than
+	// having to convert both here and there...
+
+	i.mut.Lock()
+	vals := make([]string, 0, len(i.val2id))
+	for val := range i.val2id {
+		vals = append(vals, val)
+	}
+	i.mut.Unlock()
+
+	sort.Strings(vals)
+	return vals
+}

lib/db/smallindex_test.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import "testing"
+
+func TestSmallIndex(t *testing.T) {
+	db := OpenMemory()
+	idx := newSmallIndex(db.DB, []byte{12, 34})
+
+	// ID zero should be unallocated
+	if val, ok := idx.Val(0); ok || val != nil {
+		t.Fatal("Unexpected return for nonexistent ID 0")
+	}
+
+	// A new key should get ID zero
+	if id := idx.ID([]byte("hello")); id != 0 {
+		t.Fatal("Expected 0, not", id)
+	}
+	// Looking up ID zero should work
+	if val, ok := idx.Val(0); !ok || string(val) != "hello" {
+		t.Fatalf(`Expected true, "hello", not %v, %q`, ok, val)
+	}
+
+	// Delete the key
+	idx.Delete([]byte("hello"))
+
+	// Next ID should be one
+	if id := idx.ID([]byte("key2")); id != 1 {
+		t.Fatal("Expected 1, not", id)
+	}
+
+	// Now lets create a new index instance based on what's actually serialized to the database.
+	idx = newSmallIndex(db.DB, []byte{12, 34})
+
+	// Status should be about the same as before.
+	if val, ok := idx.Val(0); ok || val != nil {
+		t.Fatal("Unexpected return for deleted ID 0")
+	}
+	if id := idx.ID([]byte("key2")); id != 1 {
+		t.Fatal("Expected 1, not", id)
+	}
+
+	// Setting "hello" again should get us ID 2, not 0 as it was originally.
+	if id := idx.ID([]byte("hello")); id != 2 {
+		t.Fatal("Expected 2, not", id)
+	}
+}

lib/model/model.go

@@ -84,7 +84,7 @@ type Model struct {
 	*suture.Supervisor
 
 	cfg               *config.Wrapper
-	db                *db.Instance
+	db                *db.Lowlevel
 	finder            *db.BlockFinder
 	progressEmitter   *ProgressEmitter
 	id                protocol.DeviceID
@@ -134,7 +134,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Instance, protectedFiles []string) *Model {
+func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) *Model {
 	m := &Model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {

lib/model/model_test.go

@@ -2618,60 +2618,6 @@ func TestIssue4357(t *testing.T) {
 	}
 }
 
-func TestScanNoDatabaseWrite(t *testing.T) {
-	// When scanning, nothing should be committed to database unless
-	// something actually changed.
-
-	db := db.OpenMemory()
-	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.StartFolder("default")
-	m.ServeBackground()
-
-	// Reach in and update the ignore matcher to one that always does
-	// reloads when asked to, instead of checking file mtimes. This is
-	// because we will be changing the files on disk often enough that the
-	// mtimes will be unreliable to determine change status.
-	m.fmut.Lock()
-	m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
-	m.fmut.Unlock()
-
-	m.SetIgnores("default", nil)
-	defer os.Remove("testdata/.stignore")
-
-	// Scan the folder twice. The second scan should be a no-op database wise
-
-	m.ScanFolder("default")
-	c0 := db.Committed()
-
-	m.ScanFolder("default")
-	c1 := db.Committed()
-
-	if c1 != c0 {
-		t.Errorf("scan should not commit data when nothing changed but %d != %d", c1, c0)
-	}
-
-	// Ignore a file we know exists. It'll be updated in the database.
-
-	m.SetIgnores("default", []string{"foo"})
-
-	m.ScanFolder("default")
-	c2 := db.Committed()
-
-	if c2 <= c1 {
-		t.Errorf("scan should commit data when something got ignored but %d <= %d", c2, c1)
-	}
-
-	// Scan again. Nothing should happen.
-
-	m.ScanFolder("default")
-	c3 := db.Committed()
-
-	if c3 != c2 {
-		t.Errorf("scan should not commit data when nothing changed (with ignores) but %d != %d", c3, c2)
-	}
-}
-
 func TestIssue2782(t *testing.T) {
 	// CheckHealth should accept a symlinked folder, when using tilde-expanded path.
 

lib/stats/device.go

@@ -21,10 +21,9 @@ type DeviceStatisticsReference struct {
 	device string
 }
 
-func NewDeviceStatisticsReference(ldb *db.Instance, device string) *DeviceStatisticsReference {
+func NewDeviceStatisticsReference(ldb *db.Lowlevel, device string) *DeviceStatisticsReference {
-	prefix := string(db.KeyTypeDeviceStatistic) + device
 	return &DeviceStatisticsReference{
-		ns:     db.NewNamespacedKV(ldb, prefix),
+		ns:     db.NewDeviceStatisticsNamespace(ldb, device),
 		device: device,
 	}
 }

lib/stats/folder.go

@@ -28,10 +28,9 @@ type LastFile struct {
 	Deleted  bool      `json:"deleted"`
 }
 
-func NewFolderStatisticsReference(ldb *db.Instance, folder string) *FolderStatisticsReference {
+func NewFolderStatisticsReference(ldb *db.Lowlevel, folder string) *FolderStatisticsReference {
-	prefix := string(db.KeyTypeFolderStatistic) + folder
 	return &FolderStatisticsReference{
-		ns:     db.NewNamespacedKV(ldb, prefix),
+		ns:     db.NewFolderStatisticsNamespace(ldb, folder),
 		folder: folder,
 	}
 }