From b50d57b7fd84fc767661c60bb3ab53d0d3582d05 Mon Sep 17 00:00:00 2001
From: Jakob Borg <jakob@kastelo.net>
Date: Wed, 10 Oct 2018 11:34:24 +0200
Subject: [PATCH] lib/db: Refactor: use a Lowlevel type underneath Instance
 (ref #5198) (#5212)

This adds a thin type that holds the state associated with the
leveldb.DB, leaving the huge Instance type more or less stateless. Also
moves some keying stuff into the DB package so that other packages need
not know the keying specifics.

(This does not, yet, fix the cmd/stindex program, in order to keep the
diff size down. Hence the keying constants are still exported.)
---
 cmd/stindex/dump.go                       |   2 +-
 cmd/stindex/dumpsize.go                   |   2 +-
 cmd/syncthing/main.go                     |  10 +-
 lib/db/benchmark_test.go                  |   2 +-
 lib/db/blockmap.go                        |  10 +-
 lib/db/blockmap_test.go                   |  10 +-
 lib/db/keyer_test.go                      |   6 +-
 lib/db/leveldb.go                         |   2 +-
 lib/db/leveldb_dbinstance.go              | 186 ++++------------------
 lib/db/leveldb_dbinstance_updateschema.go |  23 ++-
 lib/db/leveldb_test.go                    | 122 ++------------
 lib/db/leveldb_transactions.go            |   8 +-
 lib/db/lowlevel.go                        | 122 ++++++++++++++
 lib/db/meta.go                            |  25 ++-
 lib/db/namespaced.go                      |  24 ++-
 lib/db/set.go                             |  32 ++--
 lib/db/smallindex.go                      |  50 +++++-
 lib/db/smallindex_test.go                 |  52 ++++++
 lib/model/model.go                        |   4 +-
 lib/model/model_test.go                   |  54 -------
 lib/stats/device.go                       |   5 +-
 lib/stats/folder.go                       |   5 +-
 22 files changed, 382 insertions(+), 374 deletions(-)
 create mode 100644 lib/db/lowlevel.go
 create mode 100644 lib/db/smallindex_test.go

diff --git a/cmd/stindex/dump.go b/cmd/stindex/dump.go
index dc0edf53..34edbdee 100644
--- a/cmd/stindex/dump.go
+++ b/cmd/stindex/dump.go
@@ -16,7 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Instance) {
+func dump(ldb *db.Lowlevel) {
 	it := ldb.NewIterator(nil, nil)
 	for it.Next() {
 		key := it.Key()
diff --git a/cmd/stindex/dumpsize.go b/cmd/stindex/dumpsize.go
index bc26f691..71c35169 100644
--- a/cmd/stindex/dumpsize.go
+++ b/cmd/stindex/dumpsize.go
@@ -37,7 +37,7 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Instance) {
+func dumpsize(ldb *db.Lowlevel) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go
index 59d92dff..597cf529 100644
--- a/cmd/syncthing/main.go
+++ b/cmd/syncthing/main.go
@@ -712,11 +712,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	if err != nil {
 		l.Fatalln("Error opening database:", err)
 	}
+	if err := db.UpdateSchema(ldb); err != nil {
+		l.Fatalln("Database schema:", err)
+	}
 
 	if runtimeOptions.resetDeltaIdxs {
 		l.Infoln("Reinitializing delta index IDs")
-		ldb.DropLocalDeltaIndexIDs()
-		ldb.DropRemoteDeltaIndexIDs()
+		db.DropDeltaIndexIDs(ldb)
 	}
 
 	protectedFiles := []string{
@@ -737,7 +739,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// Grab the previously running version string from the database.
 
-	miscDB := db.NewNamespacedKV(ldb, string(db.KeyTypeMiscData))
+	miscDB := db.NewMiscDataNamespace(ldb)
 	prevVersion, _ := miscDB.String("prevVersion")
 
 	// Strip away prerelease/beta stuff and just compare the release
@@ -753,7 +755,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 		// Drop delta indexes in case we've changed random stuff we
 		// shouldn't have. We will resend our index on next connect.
-		ldb.DropLocalDeltaIndexIDs()
+		db.DropDeltaIndexIDs(ldb)
 
 		// Remember the new version.
 		miscDB.PutString("prevVersion", Version)
diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go
index 144b6670..500517ef 100644
--- a/lib/db/benchmark_test.go
+++ b/lib/db/benchmark_test.go
@@ -45,7 +45,7 @@ func lazyInitBenchFileSet() {
 	replace(benchS, protocol.LocalDeviceID, firstHalf)
 }
 
-func tempDB() (*db.Instance, string) {
+func tempDB() (*db.Lowlevel, string) {
 	dir, err := ioutil.TempDir("", "syncthing")
 	if err != nil {
 		panic(err)
diff --git a/lib/db/blockmap.go b/lib/db/blockmap.go
index 7e9f5f56..bb427750 100644
--- a/lib/db/blockmap.go
+++ b/lib/db/blockmap.go
@@ -22,14 +22,14 @@ var blockFinder *BlockFinder
 const maxBatchSize = 1000
 
 type BlockMap struct {
-	db     *Instance
+	db     *Lowlevel
 	folder uint32
 }
 
-func NewBlockMap(db *Instance, folder uint32) *BlockMap {
+func NewBlockMap(db *Lowlevel, folder string) *BlockMap {
 	return &BlockMap{
 		db:     db,
-		folder: folder,
+		folder: db.folderIdx.ID([]byte(folder)),
 	}
 }
 
@@ -139,10 +139,10 @@ func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte {
 }
 
 type BlockFinder struct {
-	db *Instance
+	db *Lowlevel
 }
 
-func NewBlockFinder(db *Instance) *BlockFinder {
+func NewBlockFinder(db *Lowlevel) *BlockFinder {
 	if blockFinder != nil {
 		return blockFinder
 	}
diff --git a/lib/db/blockmap_test.go b/lib/db/blockmap_test.go
index b7446272..b55b6d84 100644
--- a/lib/db/blockmap_test.go
+++ b/lib/db/blockmap_test.go
@@ -48,14 +48,14 @@ func init() {
 	}
 }
 
-func setup() (*Instance, *BlockFinder) {
+func setup() (*Lowlevel, *BlockFinder) {
 	// Setup
 
 	db := OpenMemory()
 	return db, NewBlockFinder(db)
 }
 
-func dbEmpty(db *Instance) bool {
+func dbEmpty(db *Lowlevel) bool {
 	iter := db.NewIterator(util.BytesPrefix([]byte{KeyTypeBlock}), nil)
 	defer iter.Release()
 	return !iter.Next()
@@ -68,7 +68,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal("db not empty")
 	}
 
-	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
+	m := NewBlockMap(db, "folder1")
 
 	f3.Type = protocol.FileInfoTypeDirectory
 
@@ -152,8 +152,8 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 func TestBlockFinderLookup(t *testing.T) {
 	db, f := setup()
 
-	m1 := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
-	m2 := NewBlockMap(db, db.folderIdx.ID([]byte("folder2")))
+	m1 := NewBlockMap(db, "folder1")
+	m2 := NewBlockMap(db, "folder2")
 
 	err := m1.Add([]protocol.FileInfo{f1})
 	if err != nil {
diff --git a/lib/db/keyer_test.go b/lib/db/keyer_test.go
index 291a4dd5..6ce7c983 100644
--- a/lib/db/keyer_test.go
+++ b/lib/db/keyer_test.go
@@ -16,7 +16,7 @@ func TestDeviceKey(t *testing.T) {
 	dev := []byte("device67890123456789012345678901")
 	name := []byte("name")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	key := db.keyer.GenerateDeviceFileKey(nil, fld, dev, name)
 
@@ -44,7 +44,7 @@ func TestGlobalKey(t *testing.T) {
 	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
 	name := []byte("name")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	key := db.keyer.GenerateGlobalVersionKey(nil, fld, name)
 
@@ -69,7 +69,7 @@ func TestGlobalKey(t *testing.T) {
 func TestSequenceKey(t *testing.T) {
 	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
 
-	db := OpenMemory()
+	db := newInstance(OpenMemory())
 
 	const seq = 1234567890
 	key := db.keyer.GenerateSequenceKey(nil, fld, seq)
diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go
index d2829689..4e244427 100644
--- a/lib/db/leveldb.go
+++ b/lib/db/leveldb.go
@@ -31,7 +31,7 @@ func (vl VersionList) String() string {
 // update brings the VersionList up to date with file. It returns the updated
 // VersionList, a potentially removed old FileVersion and its index, as well as
 // the index where the new FileVersion was inserted.
-func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
+func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
 	removedAt, insertedAt = -1, -1
 	for i, v := range vl.Versions {
 		if bytes.Equal(v.Device, device) {
diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go
index ead076f7..8c1a972d 100644
--- a/lib/db/leveldb_dbinstance.go
+++ b/lib/db/leveldb_dbinstance.go
@@ -10,87 +10,28 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"os"
-	"sort"
-	"strings"
-	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator)
 
-type Instance struct {
-	committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
-	*leveldb.DB
-	location  string
-	folderIdx *smallIndex
-	deviceIdx *smallIndex
-	keyer     keyer
+type instance struct {
+	*Lowlevel
+	keyer keyer
 }
 
-func Open(file string) (*Instance, error) {
-	opts := &opt.Options{
-		OpenFilesCacheCapacity: 100,
-		WriteBuffer:            4 << 20,
+func newInstance(ll *Lowlevel) *instance {
+	return &instance{
+		Lowlevel: ll,
+		keyer:    newDefaultKeyer(ll.folderIdx, ll.deviceIdx),
 	}
-
-	db, err := leveldb.OpenFile(file, opts)
-	if leveldbIsCorrupted(err) {
-		db, err = leveldb.RecoverFile(file, opts)
-	}
-	if leveldbIsCorrupted(err) {
-		// The database is corrupted, and we've tried to recover it but it
-		// didn't work. At this point there isn't much to do beyond dropping
-		// the database and reindexing...
-		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
-		if err := os.RemoveAll(file); err != nil {
-			return nil, errorSuggestion{err, "failed to delete corrupted database"}
-		}
-		db, err = leveldb.OpenFile(file, opts)
-	}
-	if err != nil {
-		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
-	}
-
-	return newDBInstance(db, file)
 }
 
-func OpenMemory() *Instance {
-	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	ldb, _ := newDBInstance(db, "<memory>")
-	return ldb
-}
-
-func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
-	i := &Instance{
-		DB:        db,
-		location:  location,
-		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
-		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
-	}
-	i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
-	err := i.updateSchema()
-	return i, err
-}
-
-// Committed returns the number of items committed to the database since startup
-func (db *Instance) Committed() int64 {
-	return atomic.LoadInt64(&db.committed)
-}
-
-// Location returns the filesystem path where the database is stored
-func (db *Instance) Location() string {
-	return db.location
-}
-
-func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
+func (db *instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -131,7 +72,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
 	}
 }
 
-func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
+func (db *instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -146,7 +87,7 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	}
 }
 
-func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
+func (db *instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -158,7 +99,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {
 	}
 }
 
-func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
+func (db *instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
 	if len(prefix) > 0 {
 		unslashedPrefix := prefix
 		if bytes.HasSuffix(prefix, []byte{'/'}) {
@@ -199,7 +140,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 	}
 }
 
-func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
+func (db *instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -226,7 +167,7 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
 	}
 }
 
-func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
+func (db *instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -271,14 +212,14 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 	}
 }
 
-func (db *Instance) getFile(key []byte) (protocol.FileInfo, bool) {
+func (db *instance) getFile(key []byte) (protocol.FileInfo, bool) {
 	if f, ok := db.getFileTrunc(key, false); ok {
 		return f.(protocol.FileInfo), true
 	}
 	return protocol.FileInfo{}, false
 }
 
-func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
+func (db *instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
 	bs, err := db.Get(key, nil)
 	if err == leveldb.ErrNotFound {
 		return nil, false
@@ -296,7 +237,7 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
 	return f, true
 }
 
-func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
+func (db *instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -304,7 +245,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 	return f, ok
 }
 
-func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
+func (db *instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
 	gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)
 
 	bs, err := t.Get(gk, nil)
@@ -325,7 +266,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
 	return gk, dk, nil, false
 }
 
-func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
+func (db *instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
 	if len(prefix) > 0 {
 		unslashedPrefix := prefix
 		if bytes.HasSuffix(prefix, []byte{'/'}) {
@@ -370,7 +311,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	}
 }
 
-func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
+func (db *instance) availability(folder, file []byte) []protocol.DeviceID {
 	k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
 	bs, err := db.Get(k, nil)
 	if err == leveldb.ErrNotFound {
@@ -401,7 +342,7 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	return devices
 }
 
-func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
+func (db *instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
 	if bytes.Equal(device, protocol.LocalDeviceID[:]) {
 		db.withNeedLocal(folder, truncate, fn)
 		return
@@ -473,7 +414,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 	}
 }
 
-func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
+func (db *instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()
 
@@ -495,31 +436,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	}
 }
 
-func (db *Instance) ListFolders() []string {
-	t := db.newReadOnlyTransaction()
-	defer t.close()
-
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
-	defer dbi.Release()
-
-	folderExists := make(map[string]bool)
-	for dbi.Next() {
-		folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
-		if ok && !folderExists[string(folder)] {
-			folderExists[string(folder)] = true
-		}
-	}
-
-	folders := make([]string, 0, len(folderExists))
-	for k := range folderExists {
-		folders = append(folders, k)
-	}
-
-	sort.Strings(folders)
-	return folders
-}
-
-func (db *Instance) dropFolder(folder []byte) {
+func (db *instance) dropFolder(folder []byte) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -537,7 +454,7 @@ func (db *Instance) dropFolder(folder []byte) {
 	}
 }
 
-func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
+func (db *instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -556,7 +473,7 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	}
 }
 
-func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
+func (db *instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -604,7 +521,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	l.Debugf("db check completed for %q", folder)
 }
 
-func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
+func (db *instance) getIndexID(device, folder []byte) protocol.IndexID {
 	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	cur, err := db.Get(key, nil)
 	if err != nil {
@@ -619,7 +536,7 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
 	return id
 }
 
-func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
+func (db *instance) setIndexID(device, folder []byte, id protocol.IndexID) {
 	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	bs, _ := id.Marshal() // marshalling can't fail
 	if err := db.Put(key, bs, nil); err != nil {
@@ -627,44 +544,15 @@ func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
 	}
 }
 
-// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
-// the database. This will cause a full index transmission on the next
-// connection.
-func (db *Instance) DropLocalDeltaIndexIDs() {
-	db.dropDeltaIndexIDs(true)
-}
-
-// DropRemoteDeltaIndexIDs removes all index IDs for the other devices than
-// the local one from the database. This will cause them to send us a full
-// index on the next connection.
-func (db *Instance) DropRemoteDeltaIndexIDs() {
-	db.dropDeltaIndexIDs(false)
-}
-
-func (db *Instance) dropDeltaIndexIDs(local bool) {
-	t := db.newReadWriteTransaction()
-	defer t.close()
-
-	dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
-	defer dbi.Release()
-
-	for dbi.Next() {
-		device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
-		if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
-			t.Delete(dbi.Key())
-		}
-	}
-}
-
-func (db *Instance) dropMtimes(folder []byte) {
+func (db *instance) dropMtimes(folder []byte) {
 	db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
 }
 
-func (db *Instance) dropFolderMeta(folder []byte) {
+func (db *instance) dropFolderMeta(folder []byte) {
 	db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
 }
 
-func (db *Instance) dropPrefix(prefix []byte) {
+func (db *instance) dropPrefix(prefix []byte) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -701,22 +589,6 @@ func unmarshalVersionList(data []byte) (VersionList, bool) {
 	return vl, true
 }
 
-// A "better" version of leveldb's errors.IsCorrupted.
-func leveldbIsCorrupted(err error) bool {
-	switch {
-	case err == nil:
-		return false
-
-	case errors.IsCorrupted(err):
-		return true
-
-	case strings.Contains(err.Error(), "corrupted"):
-		return true
-	}
-
-	return false
-}
-
 type errorSuggestion struct {
 	inner      error
 	suggestion string
diff --git a/lib/db/leveldb_dbinstance_updateschema.go b/lib/db/leveldb_dbinstance_updateschema.go
index 36d174bf..bed8c9fc 100644
--- a/lib/db/leveldb_dbinstance_updateschema.go
+++ b/lib/db/leveldb_dbinstance_updateschema.go
@@ -38,8 +38,17 @@ func (e databaseDowngradeError) Error() string {
 	return fmt.Sprintf("Syncthing %s required", e.minSyncthingVersion)
 }
 
-func (db *Instance) updateSchema() error {
-	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
+func UpdateSchema(ll *Lowlevel) error {
+	updater := &schemaUpdater{newInstance(ll)}
+	return updater.updateSchema()
+}
+
+type schemaUpdater struct {
+	*instance
+}
+
+func (db *schemaUpdater) updateSchema() error {
+	miscDB := NewMiscDataNamespace(db.Lowlevel)
 	prevVersion, _ := miscDB.Int64("dbVersion")
 
 	if prevVersion > dbVersion {
@@ -77,7 +86,7 @@ func (db *Instance) updateSchema() error {
 	return nil
 }
 
-func (db *Instance) updateSchema0to1() {
+func (db *schemaUpdater) updateSchema0to1() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -159,7 +168,7 @@ func (db *Instance) updateSchema0to1() {
 
 // updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
 // to allow iteration in sequence order (simplifies sending indexes).
-func (db *Instance) updateSchema1to2() {
+func (db *schemaUpdater) updateSchema1to2() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -178,7 +187,7 @@ func (db *Instance) updateSchema1to2() {
 }
 
 // updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
-func (db *Instance) updateSchema2to3() {
+func (db *schemaUpdater) updateSchema2to3() {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
@@ -209,7 +218,7 @@ func (db *Instance) updateSchema2to3() {
 // release candidates (dbVersion 3 and 4)
 // https://github.com/syncthing/syncthing/issues/5007
 // https://github.com/syncthing/syncthing/issues/5053
-func (db *Instance) updateSchemaTo5() {
+func (db *schemaUpdater) updateSchemaTo5() {
 	t := db.newReadWriteTransaction()
 	var nk []byte
 	for _, folderStr := range db.ListFolders() {
@@ -221,7 +230,7 @@ func (db *Instance) updateSchemaTo5() {
 	db.updateSchema2to3()
 }
 
-func (db *Instance) updateSchema5to6() {
+func (db *schemaUpdater) updateSchema5to6() {
 	// For every local file with the Invalid bit set, clear the Invalid bit and
 	// set LocalFlags = FlagLocalIgnored.
 
diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go
index 1699eccf..40ec99c0 100644
--- a/lib/db/leveldb_test.go
+++ b/lib/db/leveldb_test.go
@@ -7,107 +7,20 @@
 package db
 
 import (
-	"os"
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func TestDropIndexIDs(t *testing.T) {
-	db := OpenMemory()
-
-	d1 := []byte("device67890123456789012345678901")
-	d2 := []byte("device12345678901234567890123456")
-
-	// Set some index IDs
-
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
-	db.setIndexID(d1, []byte("foo"), 3)
-	db.setIndexID(d1, []byte("bar"), 4)
-	db.setIndexID(d2, []byte("foo"), 5)
-	db.setIndexID(d2, []byte("bar"), 6)
-
-	// Verify them
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 3 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 4 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 5 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 6 {
-		t.Fatal("fail remote 4")
-	}
-
-	// Drop the local ones, verify only they got dropped
-
-	db.DropLocalDeltaIndexIDs()
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 0 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 0 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 3 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 4 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 5 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 6 {
-		t.Fatal("fail remote 4")
-	}
-
-	// Set local ones again
-
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1)
-	db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2)
-
-	// Drop the remote ones, verify only they got dropped
-
-	db.DropRemoteDeltaIndexIDs()
-
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 {
-		t.Fatal("fail local 1")
-	}
-	if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 {
-		t.Fatal("fail local 2")
-	}
-	if db.getIndexID(d1, []byte("foo")) != 0 {
-		t.Fatal("fail remote 1")
-	}
-	if db.getIndexID(d1, []byte("bar")) != 0 {
-		t.Fatal("fail remote 2")
-	}
-	if db.getIndexID(d2, []byte("foo")) != 0 {
-		t.Fatal("fail remote 3")
-	}
-	if db.getIndexID(d2, []byte("bar")) != 0 {
-		t.Fatal("fail remote 4")
-	}
-}
-
 func TestIgnoredFiles(t *testing.T) {
 	ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons")
 	if err != nil {
 		t.Fatal(err)
 	}
-	db, _ := newDBInstance(ldb, "<memory>")
+	db := NewLowlevel(ldb, "<memory>")
+	UpdateSchema(db)
+
 	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
 
 	// The contents of the database are like this:
@@ -228,11 +141,13 @@ func TestUpdate0to3(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	db, _ := newDBInstance(ldb, "<memory>")
+
+	db := newInstance(NewLowlevel(ldb, "<memory>"))
+	updater := schemaUpdater{db}
 
 	folder := []byte(update0to3Folder)
 
-	db.updateSchema0to1()
+	updater.updateSchema0to1()
 
 	if _, ok := db.getFile(db.keyer.GenerateDeviceFileKey(nil, folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
 		t.Error("File prefixed by '/' was not removed during transition to schema 1")
@@ -242,7 +157,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Invalid file wasn't added to global list")
 	}
 
-	db.updateSchema1to2()
+	updater.updateSchema1to2()
 
 	found := false
 	db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
@@ -263,7 +178,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("Local file wasn't added to sequence bucket", err)
 	}
 
-	db.updateSchema2to3()
+	updater.updateSchema2to3()
 
 	need := map[string]protocol.FileInfo{
 		haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
@@ -288,22 +203,17 @@ func TestUpdate0to3(t *testing.T) {
 }
 
 func TestDowngrade(t *testing.T) {
-	loc := "testdata/downgrade.db"
-	db, err := Open(loc)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		db.Close()
-		os.RemoveAll(loc)
-	}()
+	db := OpenMemory()
+	UpdateSchema(db) // sets the min version etc
 
-	miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
+	// Bump the database version to something newer than we actually support
+	miscDB := NewMiscDataNamespace(db)
 	miscDB.PutInt64("dbVersion", dbVersion+1)
 	l.Infoln(dbVersion)
 
-	db.Close()
-	db, err = Open(loc)
+	// Pretend we just opened the DB and attempt to update it again
+	err := UpdateSchema(db)
+
 	if err, ok := err.(databaseDowngradeError); !ok {
 		t.Fatal("Expected error due to database downgrade, got", err)
 	} else if err.minSyncthingVersion != dbMinSyncthingVersion {
diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go
index 7b04192b..e8d4098e 100644
--- a/lib/db/leveldb_transactions.go
+++ b/lib/db/leveldb_transactions.go
@@ -8,7 +8,6 @@ package db
 
 import (
 	"bytes"
-	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -18,10 +17,10 @@ import (
 // A readOnlyTransaction represents a database snapshot.
 type readOnlyTransaction struct {
 	*leveldb.Snapshot
-	db *Instance
+	db *instance
 }
 
-func (db *Instance) newReadOnlyTransaction() readOnlyTransaction {
+func (db *instance) newReadOnlyTransaction() readOnlyTransaction {
 	snap, err := db.GetSnapshot()
 	if err != nil {
 		panic(err)
@@ -48,7 +47,7 @@ type readWriteTransaction struct {
 	*leveldb.Batch
 }
 
-func (db *Instance) newReadWriteTransaction() readWriteTransaction {
+func (db *instance) newReadWriteTransaction() readWriteTransaction {
 	t := db.newReadOnlyTransaction()
 	return readWriteTransaction{
 		readOnlyTransaction: t,
@@ -72,7 +71,6 @@ func (t readWriteTransaction) flush() {
 	if err := t.db.Write(t.Batch, nil); err != nil {
 		panic(err)
 	}
-	atomic.AddInt64(&t.db.committed, int64(t.Batch.Len()))
 }
 
 func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protocol.FileInfo) {
diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go
new file mode 100644
index 00000000..ad3451d7
--- /dev/null
+++ b/lib/db/lowlevel.go
@@ -0,0 +1,122 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"os"
+	"strings"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+const (
+	dbMaxOpenFiles = 100
+	dbWriteBuffer  = 4 << 20
+)
+
+// Lowlevel is the lowest level database interface. It has a very simple
+// purpose: hold the actual *leveldb.DB database, and the in-memory state
+// that belong to that database. In the same way that a single on disk
+// database can only be opened once, there should be only one Lowlevel for
+// any given *leveldb.DB.
+type Lowlevel struct {
+	committed int64 // atomic, must come first
+	*leveldb.DB
+	location  string
+	folderIdx *smallIndex
+	deviceIdx *smallIndex
+}
+
+// Open attempts to open the database at the given location, and runs
+// recovery on it if opening fails. Worst case, if recovery is not possible,
+// the database is erased and created from scratch.
+func Open(location string) (*Lowlevel, error) {
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: dbMaxOpenFiles,
+		WriteBuffer:            dbWriteBuffer,
+	}
+
+	db, err := leveldb.OpenFile(location, opts)
+	if leveldbIsCorrupted(err) {
+		db, err = leveldb.RecoverFile(location, opts)
+	}
+	if leveldbIsCorrupted(err) {
+		// The database is corrupted, and we've tried to recover it but it
+		// didn't work. At this point there isn't much to do beyond dropping
+		// the database and reindexing...
+		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
+		if err := os.RemoveAll(location); err != nil {
+			return nil, errorSuggestion{err, "failed to delete corrupted database"}
+		}
+		db, err = leveldb.OpenFile(location, opts)
+	}
+	if err != nil {
+		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
+	}
+	return NewLowlevel(db, location), nil
+}
+
+// OpenMemory returns a new Lowlevel referencing an in-memory database.
+func OpenMemory() *Lowlevel {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	return NewLowlevel(db, "<memory>")
+}
+
+// Location returns the filesystem path where the database is stored
+func (db *Lowlevel) Location() string {
+	return db.location
+}
+
+// ListFolders returns the list of folders currently in the database
+func (db *Lowlevel) ListFolders() []string {
+	return db.folderIdx.Values()
+}
+
+// Committed returns the number of items committed to the database since startup
+func (db *Lowlevel) Committed() int64 {
+	return atomic.LoadInt64(&db.committed)
+}
+
+func (db *Lowlevel) Put(key, val []byte, wo *opt.WriteOptions) error {
+	atomic.AddInt64(&db.committed, 1)
+	return db.DB.Put(key, val, wo)
+}
+
+func (db *Lowlevel) Delete(key []byte, wo *opt.WriteOptions) error {
+	atomic.AddInt64(&db.committed, 1)
+	return db.DB.Delete(key, wo)
+}
+
+// NewLowlevel wraps the given *leveldb.DB into a *lowlevel
+func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
+	return &Lowlevel{
+		DB:        db,
+		location:  location,
+		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
+	}
+}
+
+// A "better" version of leveldb's errors.IsCorrupted.
+func leveldbIsCorrupted(err error) bool {
+	switch {
+	case err == nil:
+		return false
+
+	case errors.IsCorrupted(err):
+		return true
+
+	case strings.Contains(err.Error(), "corrupted"):
+		return true
+	}
+
+	return false
+}
diff --git a/lib/db/meta.go b/lib/db/meta.go
index 097df0a8..dffc3f3e 100644
--- a/lib/db/meta.go
+++ b/lib/db/meta.go
@@ -20,6 +20,7 @@ type metadataTracker struct {
 	mut     sync.RWMutex
 	counts  CountsSet
 	indexes map[metaKey]int // device ID + local flags -> index in counts
+	dirty   bool
 }
 
 type metaKey struct {
@@ -55,18 +56,31 @@ func (m *metadataTracker) Marshal() ([]byte, error) {
 
 // toDB saves the marshalled metadataTracker to the given db, under the key
 // corresponding to the given folder
-func (m *metadataTracker) toDB(db *Instance, folder []byte) error {
+func (m *metadataTracker) toDB(db *instance, folder []byte) error {
 	key := db.keyer.GenerateFolderMetaKey(nil, folder)
+
+	m.mut.RLock()
+	defer m.mut.RUnlock()
+
+	if !m.dirty {
+		return nil
+	}
+
 	bs, err := m.Marshal()
 	if err != nil {
 		return err
 	}
-	return db.Put(key, bs, nil)
+	err = db.Put(key, bs, nil)
+	if err == nil {
+		m.dirty = false
+	}
+
+	return err
 }
 
 // fromDB initializes the metadataTracker from the marshalled data found in
 // the database under the key corresponding to the given folder
-func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
+func (m *metadataTracker) fromDB(db *instance, folder []byte) error {
 	key := db.keyer.GenerateFolderMetaKey(nil, folder)
 	bs, err := db.Get(key, nil)
 	if err != nil {
@@ -99,6 +113,7 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
 	}
 
 	m.mut.Lock()
+	m.dirty = true
 
 	if flags := f.FileLocalFlags(); flags == 0 {
 		// Account regular files in the zero-flags bucket.
@@ -141,6 +156,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
 	}
 
 	m.mut.Lock()
+	m.dirty = true
 
 	if flags := f.FileLocalFlags(); flags == 0 {
 		// Remove regular files from the zero-flags bucket
@@ -194,6 +210,7 @@ func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32,
 // resetAll resets all metadata for the given device
 func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
 	m.mut.Lock()
+	m.dirty = true
 	for i, c := range m.counts.Counts {
 		if bytes.Equal(c.DeviceID, dev[:]) {
 			m.counts.Counts[i] = Counts{
@@ -209,6 +226,7 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
 // sequence number
 func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
 	m.mut.Lock()
+	m.dirty = true
 
 	for i, c := range m.counts.Counts {
 		if bytes.Equal(c.DeviceID, dev[:]) {
@@ -285,6 +303,7 @@ func (m *metadataTracker) Created() time.Time {
 func (m *metadataTracker) SetCreated() {
 	m.mut.Lock()
 	m.counts.Created = time.Now().UnixNano()
+	m.dirty = true
 	m.mut.Unlock()
 }
 
diff --git a/lib/db/namespaced.go b/lib/db/namespaced.go
index edbdd127..7a26bdff 100644
--- a/lib/db/namespaced.go
+++ b/lib/db/namespaced.go
@@ -17,13 +17,13 @@ import (
 // NamespacedKV is a simple key-value store using a specific namespace within
 // a leveldb.
 type NamespacedKV struct {
-	db     *Instance
+	db     *Lowlevel
 	prefix []byte
 }
 
 // NewNamespacedKV returns a new NamespacedKV that lives in the namespace
 // specified by the prefix.
-func NewNamespacedKV(db *Instance, prefix string) *NamespacedKV {
+func NewNamespacedKV(db *Lowlevel, prefix string) *NamespacedKV {
 	return &NamespacedKV{
 		db:     db,
 		prefix: []byte(prefix),
@@ -157,3 +157,23 @@ func (n NamespacedKV) Delete(key string) {
 	keyBs := append(n.prefix, []byte(key)...)
 	n.db.Delete(keyBs, nil)
 }
+
+// Well known namespaces that can be instantiated without knowing the key
+// details.
+
+// NewDeviceStatisticsNamespace creates a KV namespace for device statistics
+// for the given device.
+func NewDeviceStatisticsNamespace(db *Lowlevel, device string) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeDeviceStatistic)+device)
+}
+
+// NewFolderStatisticsNamespace creates a KV namespace for folder statistics
+// for the given folder.
+func NewFolderStatisticsNamespace(db *Lowlevel, folder string) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeFolderStatistic)+folder)
+}
+
+// NewMiscDateNamespace creates a KV namespace for miscellaneous metadata.
+func NewMiscDataNamespace(db *Lowlevel) *NamespacedKV {
+	return NewNamespacedKV(db, string(KeyTypeMiscData))
+}
diff --git a/lib/db/set.go b/lib/db/set.go
index e979ae0f..a7c8824c 100644
--- a/lib/db/set.go
+++ b/lib/db/set.go
@@ -21,12 +21,13 @@ import (
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type FileSet struct {
 	folder   string
 	fs       fs.Filesystem
-	db       *Instance
+	db       *instance
 	blockmap *BlockMap
 	meta     *metadataTracker
 
@@ -66,12 +67,14 @@ func init() {
 	}
 }
 
-func NewFileSet(folder string, fs fs.Filesystem, db *Instance) *FileSet {
+func NewFileSet(folder string, fs fs.Filesystem, ll *Lowlevel) *FileSet {
+	db := newInstance(ll)
+
 	var s = FileSet{
 		folder:      folder,
 		fs:          fs,
 		db:          db,
-		blockmap:    NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
+		blockmap:    NewBlockMap(ll, folder),
 		meta:        newMetadataTracker(),
 		updateMutex: sync.NewMutex(),
 	}
@@ -310,7 +313,7 @@ func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
 
 func (s *FileSet) MtimeFS() *fs.MtimeFS {
 	prefix := s.db.keyer.GenerateMtimesKey(nil, []byte(s.folder))
-	kv := NewNamespacedKV(s.db, string(prefix))
+	kv := NewNamespacedKV(s.db.Lowlevel, string(prefix))
 	return fs.NewMtimeFS(s.fs, kv)
 }
 
@@ -320,15 +323,26 @@ func (s *FileSet) ListDevices() []protocol.DeviceID {
 
 // DropFolder clears out all information related to the given folder from the
 // database.
-func DropFolder(db *Instance, folder string) {
+func DropFolder(ll *Lowlevel, folder string) {
+	db := newInstance(ll)
 	db.dropFolder([]byte(folder))
 	db.dropMtimes([]byte(folder))
 	db.dropFolderMeta([]byte(folder))
-	bm := &BlockMap{
-		db:     db,
-		folder: db.folderIdx.ID([]byte(folder)),
-	}
+	bm := NewBlockMap(ll, folder)
 	bm.Drop()
+
+	// Also clean out the folder ID mapping.
+	db.folderIdx.Delete([]byte(folder))
+}
+
+// DropDeltaIndexIDs removes all delta index IDs from the database.
+// This will cause a full index transmission on the next connection.
+func DropDeltaIndexIDs(db *Lowlevel) {
+	dbi := db.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil)
+	defer dbi.Release()
+	for dbi.Next() {
+		db.Delete(dbi.Key(), nil)
+	}
 }
 
 func normalizeFilenames(fs []protocol.FileInfo) {
diff --git a/lib/db/smallindex.go b/lib/db/smallindex.go
index d171e081..9ef00f5f 100644
--- a/lib/db/smallindex.go
+++ b/lib/db/smallindex.go
@@ -8,6 +8,7 @@ package db
 
 import (
 	"encoding/binary"
+	"sort"
 
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -46,8 +47,11 @@ func (i *smallIndex) load() {
 	for it.Next() {
 		val := string(it.Value())
 		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
-		i.id2val[id] = val
-		i.val2id[val] = id
+		if val != "" {
+			// Empty value means the entry has been deleted.
+			i.id2val[id] = val
+			i.val2id[val] = id
+		}
 		if id >= i.nextID {
 			i.nextID = id + 1
 		}
@@ -96,3 +100,45 @@ func (i *smallIndex) Val(id uint32) ([]byte, bool) {
 
 	return []byte(val), true
 }
+
+func (i *smallIndex) Delete(val []byte) {
+	i.mut.Lock()
+	defer i.mut.Unlock()
+
+	// Check the reverse mapping to get the ID for the value.
+	if id, ok := i.val2id[string(val)]; ok {
+		// Generate the corresponding database key.
+		key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
+		copy(key, i.prefix)
+		binary.BigEndian.PutUint32(key[len(i.prefix):], id)
+
+		// Put an empty value into the database. This indicates that the
+		// entry does not exist any more and prevents the ID from being
+		// reused in the future.
+		i.db.Put(key, []byte{}, nil)
+
+		// Delete reverse mapping.
+		delete(i.id2val, id)
+	}
+
+	// Delete forward mapping.
+	delete(i.val2id, string(val))
+}
+
+// Values returns the set of values in the index
+func (i *smallIndex) Values() []string {
+	// In principle this method should return [][]byte because all the other
+	// methods deal in []byte keys. However, in practice, where it's used
+	// wants a []string and it's easier to just create that here rather than
+	// having to convert both here and there...
+
+	i.mut.Lock()
+	vals := make([]string, 0, len(i.val2id))
+	for val := range i.val2id {
+		vals = append(vals, val)
+	}
+	i.mut.Unlock()
+
+	sort.Strings(vals)
+	return vals
+}
diff --git a/lib/db/smallindex_test.go b/lib/db/smallindex_test.go
new file mode 100644
index 00000000..60602cd3
--- /dev/null
+++ b/lib/db/smallindex_test.go
@@ -0,0 +1,52 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import "testing"
+
+func TestSmallIndex(t *testing.T) {
+	db := OpenMemory()
+	idx := newSmallIndex(db.DB, []byte{12, 34})
+
+	// ID zero should be unallocated
+	if val, ok := idx.Val(0); ok || val != nil {
+		t.Fatal("Unexpected return for nonexistent ID 0")
+	}
+
+	// A new key should get ID zero
+	if id := idx.ID([]byte("hello")); id != 0 {
+		t.Fatal("Expected 0, not", id)
+	}
+	// Looking up ID zero should work
+	if val, ok := idx.Val(0); !ok || string(val) != "hello" {
+		t.Fatalf(`Expected true, "hello", not %v, %q`, ok, val)
+	}
+
+	// Delete the key
+	idx.Delete([]byte("hello"))
+
+	// Next ID should be one
+	if id := idx.ID([]byte("key2")); id != 1 {
+		t.Fatal("Expected 1, not", id)
+	}
+
+	// Now lets create a new index instance based on what's actually serialized to the database.
+	idx = newSmallIndex(db.DB, []byte{12, 34})
+
+	// Status should be about the same as before.
+	if val, ok := idx.Val(0); ok || val != nil {
+		t.Fatal("Unexpected return for deleted ID 0")
+	}
+	if id := idx.ID([]byte("key2")); id != 1 {
+		t.Fatal("Expected 1, not", id)
+	}
+
+	// Setting "hello" again should get us ID 2, not 0 as it was originally.
+	if id := idx.ID([]byte("hello")); id != 2 {
+		t.Fatal("Expected 2, not", id)
+	}
+}
diff --git a/lib/model/model.go b/lib/model/model.go
index 8eb49c8b..21b12add 100644
--- a/lib/model/model.go
+++ b/lib/model/model.go
@@ -84,7 +84,7 @@ type Model struct {
 	*suture.Supervisor
 
 	cfg               *config.Wrapper
-	db                *db.Instance
+	db                *db.Lowlevel
 	finder            *db.BlockFinder
 	progressEmitter   *ProgressEmitter
 	id                protocol.DeviceID
@@ -134,7 +134,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Instance, protectedFiles []string) *Model {
+func NewModel(cfg *config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) *Model {
 	m := &Model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {
diff --git a/lib/model/model_test.go b/lib/model/model_test.go
index c102af0f..f79b2e2e 100644
--- a/lib/model/model_test.go
+++ b/lib/model/model_test.go
@@ -2618,60 +2618,6 @@ func TestIssue4357(t *testing.T) {
 	}
 }
 
-func TestScanNoDatabaseWrite(t *testing.T) {
-	// When scanning, nothing should be committed to database unless
-	// something actually changed.
-
-	db := db.OpenMemory()
-	m := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.StartFolder("default")
-	m.ServeBackground()
-
-	// Reach in and update the ignore matcher to one that always does
-	// reloads when asked to, instead of checking file mtimes. This is
-	// because we will be changing the files on disk often enough that the
-	// mtimes will be unreliable to determine change status.
-	m.fmut.Lock()
-	m.folderIgnores["default"] = ignore.New(defaultFs, ignore.WithCache(true), ignore.WithChangeDetector(newAlwaysChanged()))
-	m.fmut.Unlock()
-
-	m.SetIgnores("default", nil)
-	defer os.Remove("testdata/.stignore")
-
-	// Scan the folder twice. The second scan should be a no-op database wise
-
-	m.ScanFolder("default")
-	c0 := db.Committed()
-
-	m.ScanFolder("default")
-	c1 := db.Committed()
-
-	if c1 != c0 {
-		t.Errorf("scan should not commit data when nothing changed but %d != %d", c1, c0)
-	}
-
-	// Ignore a file we know exists. It'll be updated in the database.
-
-	m.SetIgnores("default", []string{"foo"})
-
-	m.ScanFolder("default")
-	c2 := db.Committed()
-
-	if c2 <= c1 {
-		t.Errorf("scan should commit data when something got ignored but %d <= %d", c2, c1)
-	}
-
-	// Scan again. Nothing should happen.
-
-	m.ScanFolder("default")
-	c3 := db.Committed()
-
-	if c3 != c2 {
-		t.Errorf("scan should not commit data when nothing changed (with ignores) but %d != %d", c3, c2)
-	}
-}
-
 func TestIssue2782(t *testing.T) {
 	// CheckHealth should accept a symlinked folder, when using tilde-expanded path.
 
diff --git a/lib/stats/device.go b/lib/stats/device.go
index 6ba8d8b1..4817882c 100644
--- a/lib/stats/device.go
+++ b/lib/stats/device.go
@@ -21,10 +21,9 @@ type DeviceStatisticsReference struct {
 	device string
 }
 
-func NewDeviceStatisticsReference(ldb *db.Instance, device string) *DeviceStatisticsReference {
-	prefix := string(db.KeyTypeDeviceStatistic) + device
+func NewDeviceStatisticsReference(ldb *db.Lowlevel, device string) *DeviceStatisticsReference {
 	return &DeviceStatisticsReference{
-		ns:     db.NewNamespacedKV(ldb, prefix),
+		ns:     db.NewDeviceStatisticsNamespace(ldb, device),
 		device: device,
 	}
 }
diff --git a/lib/stats/folder.go b/lib/stats/folder.go
index 5515a02c..39aae681 100644
--- a/lib/stats/folder.go
+++ b/lib/stats/folder.go
@@ -28,10 +28,9 @@ type LastFile struct {
 	Deleted  bool      `json:"deleted"`
 }
 
-func NewFolderStatisticsReference(ldb *db.Instance, folder string) *FolderStatisticsReference {
-	prefix := string(db.KeyTypeFolderStatistic) + folder
+func NewFolderStatisticsReference(ldb *db.Lowlevel, folder string) *FolderStatisticsReference {
 	return &FolderStatisticsReference{
-		ns:     db.NewNamespacedKV(ldb, prefix),
+		ns:     db.NewFolderStatisticsNamespace(ldb, folder),
 		folder: folder,
 	}
 }