perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Do all update operations on a single item at once (#5441)

Browse Source 
To do so the BlockMap struct has been removed. It behaves like any other prefixed
part of the database, but was not integrated in the recent keyer refactor. Now
the database is only flushed when files are in a consistent state.
This commit is contained in:
Simon Frei
2019-01-23 10:22:33 +01:00
committed by Jakob Borg
parent 6421693cce
commit 42bd42df5a
5 changed files with 152 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
lib/db/blockmap_test.go
View File

@@ -7,6 +7,7 @@
package db
import (
"encoding/binary"
"testing"
"github.com/syncthing/syncthing/lib/protocol"
@@ -48,19 +49,53 @@ func init() {
}
}
func setup() (*Lowlevel, *BlockFinder) {
func setup() (*instance, *BlockFinder) {
// Setup
db := OpenMemory()
return db, NewBlockFinder(db)
return newInstance(db), NewBlockFinder(db)
}
func dbEmpty(db *Lowlevel) bool {
func dbEmpty(db *instance) bool {
iter := db.NewIterator(util.BytesPrefix([]byte{KeyTypeBlock}), nil)
defer iter.Release()
return !iter.Next()
}
func addToBlockMap(db *instance, folder []byte, fs []protocol.FileInfo) {
t := db.newReadWriteTransaction()
defer t.close()
var keyBuf []byte
blockBuf := make([]byte, 4)
for _, f := range fs {
if !f.IsDirectory() && !f.IsDeleted() && !f.IsInvalid() {
name := []byte(f.Name)
for i, block := range f.Blocks {
binary.BigEndian.PutUint32(blockBuf, uint32(i))
keyBuf = t.db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
t.Put(keyBuf, blockBuf)
}
}
}
}
func discardFromBlockMap(db *instance, folder []byte, fs []protocol.FileInfo) {
t := db.newReadWriteTransaction()
defer t.close()
var keyBuf []byte
for _, ef := range fs {
if !ef.IsDirectory() && !ef.IsDeleted() && !ef.IsInvalid() {
name := []byte(ef.Name)
for _, block := range ef.Blocks {
keyBuf = t.db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name)
t.Delete(keyBuf)
}
}
}
}
func TestBlockMapAddUpdateWipe(t *testing.T) {
db, f := setup()
@@ -68,14 +103,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
t.Fatal("db not empty")
}
m := NewBlockMap(db, "folder1")
folder := []byte("folder1")
f3.Type = protocol.FileInfoTypeDirectory
err := m.Add([]protocol.FileInfo{f1, f2, f3})
if err != nil {
t.Fatal(err)
}
addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3})
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
if folder != "folder1" || file != "f1" || index != 0 {
@@ -96,14 +128,12 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
return true
})
discardFromBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3})
f1.Deleted = true
f2.LocalFlags = protocol.FlagLocalMustRescan // one of the invalid markers
// Should remove
err = m.Update([]protocol.FileInfo{f1, f2, f3})
if err != nil {
t.Fatal(err)
}
addToBlockMap(db, folder, []protocol.FileInfo{f1, f2, f3})
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
t.Fatal("Unexpected block")
@@ -122,20 +152,14 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
return true
})
err = m.Drop()
if err != nil {
t.Fatal(err)
}
db.dropFolder(folder)
if !dbEmpty(db) {
t.Fatal("db not empty")
}
// Should not add
err = m.Add([]protocol.FileInfo{f1, f2})
if err != nil {
t.Fatal(err)
}
addToBlockMap(db, folder, []protocol.FileInfo{f1, f2})
if !dbEmpty(db) {
t.Fatal("db not empty")
@@ -152,17 +176,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
func TestBlockFinderLookup(t *testing.T) {
db, f := setup()
m1 := NewBlockMap(db, "folder1")
m2 := NewBlockMap(db, "folder2")
folder1 := []byte("folder1")
folder2 := []byte("folder2")
err := m1.Add([]protocol.FileInfo{f1})
if err != nil {
t.Fatal(err)
}
err = m2.Add([]protocol.FileInfo{f1})
if err != nil {
t.Fatal(err)
}
addToBlockMap(db, folder1, []protocol.FileInfo{f1})
addToBlockMap(db, folder2, []protocol.FileInfo{f1})
counter := 0
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
@@ -186,12 +204,11 @@ func TestBlockFinderLookup(t *testing.T) {
t.Fatal("Incorrect count", counter)
}
discardFromBlockMap(db, folder1, []protocol.FileInfo{f1})
f1.Deleted = true
err = m1.Update([]protocol.FileInfo{f1})
if err != nil {
t.Fatal(err)
}
addToBlockMap(db, folder1, []protocol.FileInfo{f1})
counter = 0
f.Iterate(folders, f1.Blocks[0].Hash, func(folder, file string, index int32) bool {