lib/db: Refactor key handling (ref #5198) (#5199)

This breaks out the key generation stuff into a separate type. It's cleaner on its own, and it prepares for future stuff.
2018-09-18 10:41:06 +02:00
parent 797a999585
commit 6a87aac84f
10 changed files with 465 additions and 371 deletions
--- a/lib/db/leveldb_dbinstance.go
+++ b/lib/db/leveldb_dbinstance.go
@@ -16,7 +16,6 @@ import (
 	"sync/atomic"

 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
@@ -33,18 +32,9 @@ type Instance struct {
 	location  string
 	folderIdx *smallIndex
 	deviceIdx *smallIndex
+	keyer     keyer
 }

-const (
-	keyPrefixLen   = 1
-	keyFolderLen   = 4 // indexed
-	keyDeviceLen   = 4 // indexed
-	keySequenceLen = 8
-	keyHashLen     = 32
-
-	maxInt64 int64 = 1<<63 - 1
-)
-
 func Open(file string) (*Instance, error) {
 	opts := &opt.Options{
 		OpenFilesCacheCapacity: 100,
@@ -80,11 +70,12 @@ func OpenMemory() *Instance {

 func newDBInstance(db *leveldb.DB, location string) (*Instance, error) {
 	i := &Instance{
-		DB:       db,
-		location: location,
+		DB:        db,
+		location:  location,
+		folderIdx: newSmallIndex(db, []byte{KeyTypeFolderIdx}),
+		deviceIdx: newSmallIndex(db, []byte{KeyTypeDeviceIdx}),
 	}
-	i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
-	i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
+	i.keyer = newDefaultKeyer(i.folderIdx, i.deviceIdx)
 	err := i.updateSchema()
 	return i, err
 }
@@ -107,7 +98,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m
 	var gk []byte
 	for _, f := range fs {
 		name := []byte(f.Name)
-		fk = db.deviceKeyInto(fk, folder, device, name)
+		fk = db.keyer.GenerateDeviceFileKey(fk, folder, device, name)

 		// Get and unmarshal the file entry. If it doesn't exist or can't be
 		// unmarshalled we'll add it as a new entry.
@@ -131,7 +122,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, m

 		t.insertFile(fk, folder, device, f)

-		gk = db.globalKeyInto(gk, folder, name)
+		gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 		t.updateGlobal(gk, folder, device, f, meta)

 		// Write out and reuse the batch every few records, to avoid the batch
@@ -147,8 +138,8 @@ func (db *Instance) addSequences(folder []byte, fs []protocol.FileInfo) {
 	var sk []byte
 	var dk []byte
 	for _, f := range fs {
-		sk = db.sequenceKeyInto(sk, folder, f.Sequence)
-		dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
+		sk = db.keyer.GenerateSequenceKey(sk, folder, f.Sequence)
+		dk = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], []byte(f.Name))
 		t.Put(sk, dk)
 		l.Debugf("adding sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
 		t.checkFlush()
@@ -161,7 +152,7 @@ func (db *Instance) removeSequences(folder []byte, fs []protocol.FileInfo) {

 	var sk []byte
 	for _, f := range fs {
-		t.Delete(db.sequenceKeyInto(sk, folder, f.Sequence))
+		t.Delete(db.keyer.GenerateSequenceKey(sk, folder, f.Sequence))
 		l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, f.Sequence, f.Name)
 		t.checkFlush()
 	}
@@ -176,7 +167,7 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 			prefix = append(prefix, '/')
 		}

-		if f, ok := db.getFileTrunc(db.deviceKey(folder, device, unslashedPrefix), true); ok && !fn(f) {
+		if f, ok := db.getFileTrunc(db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix), true); ok && !fn(f) {
 			return
 		}
 	}
@@ -184,11 +175,11 @@ func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn It
 	t := db.newReadOnlyTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, prefix)[:keyPrefixLen+keyFolderLen+keyDeviceLen+len(prefix)]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)), nil)
 	defer dbi.Release()

 	for dbi.Next() {
-		name := db.deviceKeyName(dbi.Key())
+		name := db.keyer.NameFromDeviceFileKey(dbi.Key())
 		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
 			return
 		}
@@ -212,13 +203,13 @@ func (db *Instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
 	t := db.newReadOnlyTransaction()
 	defer t.close()

-	dbi := t.NewIterator(&util.Range{Start: db.sequenceKey(folder, startSeq), Limit: db.sequenceKey(folder, maxInt64)}, nil)
+	dbi := t.NewIterator(&util.Range{Start: db.keyer.GenerateSequenceKey(nil, folder, startSeq), Limit: db.keyer.GenerateSequenceKey(nil, folder, maxInt64)}, nil)
 	defer dbi.Release()

 	for dbi.Next() {
 		f, ok := db.getFile(dbi.Value())
 		if !ok {
-			l.Debugln("missing file for sequence number", db.sequenceKeySequence(dbi.Key()))
+			l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
 			continue
 		}

@@ -239,13 +230,19 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 	t := db.newReadWriteTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName()), nil)
 	defer dbi.Release()

 	var gk []byte

 	for dbi.Next() {
-		device := db.deviceKeyDevice(dbi.Key())
+		device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
+		if !ok {
+			// Not having the device in the index is bad. Clear it.
+			t.Delete(dbi.Key())
+			t.checkFlush()
+			continue
+		}
 		var f FileInfoTruncated
 		// The iterator function may keep a reference to the unmarshalled
 		// struct, which in turn references the buffer it was unmarshalled
@@ -261,7 +258,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 		case "", ".", "..", "/": // A few obviously invalid filenames
 			l.Infof("Dropping invalid filename %q from database", f.Name)
 			name := []byte(f.Name)
-			gk = db.globalKeyInto(gk, folder, name)
+			gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			t.removeFromGlobal(gk, folder, device, name, nil)
 			t.Delete(dbi.Key())
 			t.checkFlush()
@@ -308,7 +305,7 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 }

 func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
-	gk = db.globalKeyInto(gk, folder, file)
+	gk = db.keyer.GenerateGlobalVersionKey(gk, folder, file)

 	bs, err := t.Get(gk, nil)
 	if err != nil {
@@ -320,7 +317,7 @@ func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []
 		return gk, dk, nil, false
 	}

-	dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
+	dk = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, file)
 	if fi, ok := db.getFileTrunc(dk, truncate); ok {
 		return gk, dk, fi, true
 	}
@@ -345,12 +342,12 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	t := db.newReadOnlyTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, prefix)), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)), nil)
 	defer dbi.Release()

 	var fk []byte
 	for dbi.Next() {
-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
 			return
 		}
@@ -360,7 +357,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 			continue
 		}

-		fk = db.deviceKeyInto(fk, folder, vl.Versions[0].Device, name)
+		fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[0].Device, name)

 		f, ok := db.getFileTrunc(fk, truncate)
 		if !ok {
@@ -374,7 +371,7 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 }

 func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
-	k := db.globalKey(folder, file)
+	k := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
 	bs, err := db.Get(k, nil)
 	if err == leveldb.ErrNotFound {
 		return nil
@@ -413,7 +410,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 	t := db.newReadOnlyTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()

 	var fk []byte
@@ -431,7 +428,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 			continue
 		}

-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		needVersion := vl.Versions[0].Version
 		needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)

@@ -446,7 +443,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 				continue
 			}

-			fk = db.deviceKeyInto(fk, folder, vl.Versions[i].Device, name)
+			fk = db.keyer.GenerateDeviceFileKey(fk, folder, vl.Versions[i].Device, name)
 			bs, err := t.Get(fk, nil)
 			if err != nil {
 				l.Debugln("surprise error:", err)
@@ -480,7 +477,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	t := db.newReadOnlyTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()

 	var dk []byte
@@ -488,7 +485,7 @@ func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
 	var f FileIntf
 	var ok bool
 	for dbi.Next() {
-		gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
+		gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
 		if !ok {
 			continue
 		}
@@ -507,7 +504,7 @@ func (db *Instance) ListFolders() []string {

 	folderExists := make(map[string]bool)
 	for dbi.Next() {
-		folder, ok := db.globalKeyFolder(dbi.Key())
+		folder, ok := db.keyer.FolderFromGlobalVersionKey(dbi.Key())
 		if ok && !folderExists[string(folder)] {
 			folderExists[string(folder)] = true
 		}
@@ -528,13 +525,13 @@ func (db *Instance) dropFolder(folder []byte) {

 	for _, key := range [][]byte{
 		// Remove all items related to the given folder from the device->file bucket
-		db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil).WithoutName(),
 		// Remove all sequences related to the folder
-		db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateSequenceKey(nil, []byte(folder), 0).WithoutSequence(),
 		// Remove all items related to the given folder from the global bucket
-		db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName(),
 		// Remove all needs related to the folder
-		db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
+		db.keyer.GenerateNeedFileKey(nil, folder, nil).WithoutName(),
 	} {
 		t.deleteKeyPrefix(key)
 	}
@@ -544,15 +541,15 @@ func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	t := db.newReadWriteTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.deviceKey(folder, device, nil)), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateDeviceFileKey(nil, folder, device, nil)), nil)
 	defer dbi.Release()

 	var gk []byte

 	for dbi.Next() {
 		key := dbi.Key()
-		name := db.deviceKeyName(key)
-		gk = db.globalKeyInto(gk, folder, name)
+		name := db.keyer.NameFromDeviceFileKey(key)
+		gk = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 		t.removeFromGlobal(gk, folder, device, name, meta)
 		t.Delete(key)
 		t.checkFlush()
@@ -563,7 +560,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()

-	dbi := t.NewIterator(util.BytesPrefix(db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
+	dbi := t.NewIterator(util.BytesPrefix(db.keyer.GenerateGlobalVersionKey(nil, folder, nil).WithoutName()), nil)
 	defer dbi.Release()

 	var fk []byte
@@ -578,10 +575,10 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 		// there are global entries pointing to no longer existing files. Here
 		// we find those and clear them out.

-		name := db.globalKeyName(dbi.Key())
+		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
 		var newVL VersionList
 		for i, version := range vl.Versions {
-			fk = db.deviceKeyInto(fk, folder, version.Device, name)
+			fk = db.keyer.GenerateDeviceFileKey(fk, folder, version.Device, name)
 			_, err := t.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
 				continue
@@ -607,110 +604,8 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
 	l.Debugf("db check completed for %q", folder)
 }

-// deviceKey returns a byte slice encoding the following information:
-//	   keyTypeDevice (1 byte)
-//	   folder (4 bytes)
-//	   device (4 bytes)
-//	   name (variable size)
-func (db *Instance) deviceKey(folder, device, file []byte) []byte {
-	return db.deviceKeyInto(nil, folder, device, file)
-}
-
-func (db *Instance) deviceKeyInto(k, folder, device, file []byte) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + keyDeviceLen + len(file)
-	k = resize(k, reqLen)
-	k[0] = KeyTypeDevice
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
-	binary.BigEndian.PutUint32(k[keyPrefixLen+keyFolderLen:], db.deviceIdx.ID(device))
-	copy(k[keyPrefixLen+keyFolderLen+keyDeviceLen:], file)
-	return k
-}
-
-// deviceKeyName returns the device ID from the key
-func (db *Instance) deviceKeyName(key []byte) []byte {
-	return key[keyPrefixLen+keyFolderLen+keyDeviceLen:]
-}
-
-// deviceKeyFolder returns the folder name from the key
-func (db *Instance) deviceKeyFolder(key []byte) []byte {
-	folder, ok := db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-	if !ok {
-		panic("bug: lookup of nonexistent folder ID")
-	}
-	return folder
-}
-
-// deviceKeyDevice returns the device ID from the key
-func (db *Instance) deviceKeyDevice(key []byte) []byte {
-	device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen+keyFolderLen:]))
-	if !ok {
-		panic("bug: lookup of nonexistent device ID")
-	}
-	return device
-}
-
-// globalKey returns a byte slice encoding the following information:
-//	   keyTypeGlobal (1 byte)
-//	   folder (4 bytes)
-//	   name (variable size)
-func (db *Instance) globalKey(folder, file []byte) []byte {
-	return db.globalKeyInto(nil, folder, file)
-}
-
-func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + len(file)
-	gk = resize(gk, reqLen)
-	gk[0] = KeyTypeGlobal
-	binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
-	copy(gk[keyPrefixLen+keyFolderLen:], file)
-	return gk
-}
-
-// globalKeyName returns the filename from the key
-func (db *Instance) globalKeyName(key []byte) []byte {
-	return key[keyPrefixLen+keyFolderLen:]
-}
-
-// globalKeyFolder returns the folder name from the key
-func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
-	return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-}
-
-// needKey is a globalKey with a different prefix
-func (db *Instance) needKey(folder, file []byte) []byte {
-	return db.needKeyInto(nil, folder, file)
-}
-
-func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
-	k = db.globalKeyInto(k, folder, file)
-	k[0] = KeyTypeNeed
-	return k
-}
-
-// sequenceKey returns a byte slice encoding the following information:
-//	   KeyTypeSequence (1 byte)
-//	   folder (4 bytes)
-//	   sequence number (8 bytes)
-func (db *Instance) sequenceKey(folder []byte, seq int64) []byte {
-	return db.sequenceKeyInto(nil, folder, seq)
-}
-
-func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
-	reqLen := keyPrefixLen + keyFolderLen + keySequenceLen
-	k = resize(k, reqLen)
-	k[0] = KeyTypeSequence
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
-	binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
-	return k
-}
-
-// sequenceKeySequence returns the sequence number from the key
-func (db *Instance) sequenceKeySequence(key []byte) int64 {
-	return int64(binary.BigEndian.Uint64(key[keyPrefixLen+keyFolderLen:]))
-}
-
 func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
-	key := db.indexIDKey(device, folder)
+	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	cur, err := db.Get(key, nil)
 	if err != nil {
 		return 0
@@ -725,44 +620,13 @@ func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
 }

 func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
-	key := db.indexIDKey(device, folder)
+	key := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	bs, _ := id.Marshal() // marshalling can't fail
 	if err := db.Put(key, bs, nil); err != nil {
 		panic("storing index ID: " + err.Error())
 	}
 }

-func (db *Instance) indexIDKey(device, folder []byte) []byte {
-	k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen)
-	k[0] = KeyTypeIndexID
-	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device))
-	binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder))
-	return k
-}
-
-func (db *Instance) indexIDDevice(key []byte) []byte {
-	device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
-	if !ok {
-		// uuh ...
-		return nil
-	}
-	return device
-}
-
-func (db *Instance) mtimesKey(folder []byte) []byte {
-	prefix := make([]byte, 5) // key type + 4 bytes folder idx number
-	prefix[0] = KeyTypeVirtualMtime
-	binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
-	return prefix
-}
-
-func (db *Instance) folderMetaKey(folder []byte) []byte {
-	prefix := make([]byte, 5) // key type + 4 bytes folder idx number
-	prefix[0] = KeyTypeFolderMeta
-	binary.BigEndian.PutUint32(prefix[1:], db.folderIdx.ID(folder))
-	return prefix
-}
-
 // DropLocalDeltaIndexIDs removes all index IDs for the local device ID from
 // the database. This will cause a full index transmission on the next
 // connection.
@@ -785,7 +649,7 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
 	defer dbi.Release()

 	for dbi.Next() {
-		device := db.indexIDDevice(dbi.Key())
+		device, _ := db.keyer.DeviceFromIndexIDKey(dbi.Key())
 		if bytes.Equal(device, protocol.LocalDeviceID[:]) == local {
 			t.Delete(dbi.Key())
 		}
@@ -793,11 +657,11 @@ func (db *Instance) dropDeltaIndexIDs(local bool) {
 }

 func (db *Instance) dropMtimes(folder []byte) {
-	db.dropPrefix(db.mtimesKey(folder))
+	db.dropPrefix(db.keyer.GenerateMtimesKey(nil, folder))
 }

 func (db *Instance) dropFolderMeta(folder []byte) {
-	db.dropPrefix(db.folderMetaKey(folder))
+	db.dropPrefix(db.keyer.GenerateFolderMetaKey(nil, folder))
 }

 func (db *Instance) dropPrefix(prefix []byte) {
@@ -853,99 +717,6 @@ func leveldbIsCorrupted(err error) bool {
 	return false
 }

-// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
-// fast lookups in both directions and persists to the database. Don't use for
-// storing more items than fit comfortably in RAM.
-type smallIndex struct {
-	db     *Instance
-	prefix []byte
-	id2val map[uint32]string
-	val2id map[string]uint32
-	nextID uint32
-	mut    sync.Mutex
-}
-
-func newSmallIndex(db *Instance, prefix []byte) *smallIndex {
-	idx := &smallIndex{
-		db:     db,
-		prefix: prefix,
-		id2val: make(map[uint32]string),
-		val2id: make(map[string]uint32),
-		mut:    sync.NewMutex(),
-	}
-	idx.load()
-	return idx
-}
-
-// load iterates over the prefix space in the database and populates the in
-// memory maps.
-func (i *smallIndex) load() {
-	tr := i.db.newReadOnlyTransaction()
-	it := tr.NewIterator(util.BytesPrefix(i.prefix), nil)
-	for it.Next() {
-		val := string(it.Value())
-		id := binary.BigEndian.Uint32(it.Key()[len(i.prefix):])
-		i.id2val[id] = val
-		i.val2id[val] = id
-		if id >= i.nextID {
-			i.nextID = id + 1
-		}
-	}
-	it.Release()
-	tr.close()
-}
-
-// ID returns the index number for the given byte slice, allocating a new one
-// and persisting this to the database if necessary.
-func (i *smallIndex) ID(val []byte) uint32 {
-	i.mut.Lock()
-	// intentionally avoiding defer here as we want this call to be as fast as
-	// possible in the general case (folder ID already exists). The map lookup
-	// with the conversion of []byte to string is compiler optimized to not
-	// copy the []byte, which is why we don't assign it to a temp variable
-	// here.
-	if id, ok := i.val2id[string(val)]; ok {
-		i.mut.Unlock()
-		return id
-	}
-
-	id := i.nextID
-	i.nextID++
-
-	valStr := string(val)
-	i.val2id[valStr] = id
-	i.id2val[id] = valStr
-
-	key := make([]byte, len(i.prefix)+8) // prefix plus uint32 id
-	copy(key, i.prefix)
-	binary.BigEndian.PutUint32(key[len(i.prefix):], id)
-	i.db.Put(key, val, nil)
-
-	i.mut.Unlock()
-	return id
-}
-
-// Val returns the value for the given index number, or (nil, false) if there
-// is no such index number.
-func (i *smallIndex) Val(id uint32) ([]byte, bool) {
-	i.mut.Lock()
-	val, ok := i.id2val[id]
-	i.mut.Unlock()
-	if !ok {
-		return nil, false
-	}
-
-	return []byte(val), true
-}
-
-// resize returns a byte array of length reqLen, reusing k if possible
-func resize(k []byte, reqLen int) []byte {
-	if cap(k) < reqLen {
-		return make([]byte, reqLen)
-	}
-	return k[:reqLen]
-}
-
 type errorSuggestion struct {
 	inner      error
 	suggestion string