perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The Great Rewrite (fixes #36, #61, #94, #101)

Browse Source 
Rewrite of the file model and pulling mechanism. Needs lots of cleanup
and bugfixes, now...
This commit is contained in:
Jakob Borg
2014-03-28 14:36:57 +01:00
parent 3700eb1e61
commit f87b1520e8
47 changed files with 2137 additions and 1902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
files/debug.go Normal file
View File

@@ -0,0 +1,12 @@
package files
import (
"log"
"os"
"strings"
)
var (
dlog = log.New(os.Stderr, "files: ", log.Lmicroseconds|log.Lshortfile)
debug = strings.Contains(os.Getenv("STTRACE"), "files")
)

324
files/set.go Normal file
View File

@@ -0,0 +1,324 @@
// Package files provides a set type to track local/remote files with newness checks.
package files
import (
"crypto/md5"
"sync"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
type key struct {
Name string
Version uint64
Modified int64
Hash [md5.Size]byte
}
type fileRecord struct {
Usage int
File scanner.File
}
type bitset uint64
func keyFor(f scanner.File) key {
h := md5.New()
for _, b := range f.Blocks {
h.Write(b.Hash)
}
return key{
Name: f.Name,
Version: f.Version,
Modified: f.Modified,
Hash: md5.Sum(nil),
}
}
func (a key) newerThan(b key) bool {
if a.Version != b.Version {
return a.Version > b.Version
}
if a.Modified != b.Modified {
return a.Modified > b.Modified
}
for i := 0; i < md5.Size; i++ {
if a.Hash[i] != b.Hash[i] {
return a.Hash[i] > b.Hash[i]
}
}
return false
}
type Set struct {
sync.Mutex
files map[key]fileRecord
remoteKey [64]map[string]key
changes [64]uint64
globalAvailability map[string]bitset
globalKey map[string]key
}
func NewSet() *Set {
var m = Set{
files: make(map[key]fileRecord),
globalAvailability: make(map[string]bitset),
globalKey: make(map[string]key),
}
return &m
}
func (m *Set) Replace(id uint, fs []scanner.File) {
if debug {
dlog.Printf("Replace(%d, [%d])", id, len(fs))
}
if id > 63 {
panic("Connection ID must be in the range 0 - 63 inclusive")
}
m.Lock()
if len(fs) == 0 || !m.equals(id, fs) {
m.changes[id]++
m.replace(id, fs)
}
m.Unlock()
}
func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
if debug {
dlog.Printf("ReplaceWithDelete(%d, [%d])", id, len(fs))
}
if id > 63 {
panic("Connection ID must be in the range 0 - 63 inclusive")
}
m.Lock()
if len(fs) == 0 || !m.equals(id, fs) {
m.changes[id]++
var nf = make(map[string]key, len(fs))
for _, f := range fs {
nf[f.Name] = keyFor(f)
}
// For previously existing files not in the list, add them to the list
// with the relevant delete flags etc set. Previously existing files
// with the delete bit already set are not modified.
for _, ck := range m.remoteKey[cid.LocalID] {
if _, ok := nf[ck.Name]; !ok {
cf := m.files[ck].File
if cf.Flags&protocol.FlagDeleted != protocol.FlagDeleted {
cf.Flags = protocol.FlagDeleted
cf.Blocks = nil
cf.Size = 0
cf.Version = lamport.Default.Tick(cf.Version)
}
fs = append(fs, cf)
if debug {
dlog.Println("deleted:", ck.Name)
}
}
}
m.replace(id, fs)
}
m.Unlock()
}
func (m *Set) Update(id uint, fs []scanner.File) {
if debug {
dlog.Printf("Update(%d, [%d])", id, len(fs))
}
m.Lock()
m.update(id, fs)
m.changes[id]++
m.Unlock()
}
func (m *Set) Need(id uint) []scanner.File {
if debug {
dlog.Printf("Need(%d)", id)
}
var fs []scanner.File
m.Lock()
for name, gk := range m.globalKey {
if gk.newerThan(m.remoteKey[id][name]) {
fs = append(fs, m.files[gk].File)
}
}
m.Unlock()
return fs
}
func (m *Set) Have(id uint) []scanner.File {
if debug {
dlog.Printf("Have(%d)", id)
}
var fs []scanner.File
m.Lock()
for _, rk := range m.remoteKey[id] {
fs = append(fs, m.files[rk].File)
}
m.Unlock()
return fs
}
func (m *Set) Global() []scanner.File {
if debug {
dlog.Printf("Global()")
}
var fs []scanner.File
m.Lock()
for _, rk := range m.globalKey {
fs = append(fs, m.files[rk].File)
}
m.Unlock()
return fs
}
func (m *Set) Get(id uint, file string) scanner.File {
m.Lock()
defer m.Unlock()
if debug {
dlog.Printf("Get(%d, %q)", id, file)
}
return m.files[m.remoteKey[id][file]].File
}
func (m *Set) GetGlobal(file string) scanner.File {
m.Lock()
defer m.Unlock()
if debug {
dlog.Printf("GetGlobal(%q)", file)
}
return m.files[m.globalKey[file]].File
}
func (m *Set) Availability(name string) bitset {
m.Lock()
defer m.Unlock()
av := m.globalAvailability[name]
if debug {
dlog.Printf("Availability(%q) = %0x", name, av)
}
return av
}
func (m *Set) Changes(id uint) uint64 {
m.Lock()
defer m.Unlock()
if debug {
dlog.Printf("Changes(%d)", id)
}
return m.changes[id]
}
func (m *Set) equals(id uint, fs []scanner.File) bool {
curWithoutDeleted := make(map[string]key)
for _, k := range m.remoteKey[id] {
f := m.files[k].File
if f.Flags&protocol.FlagDeleted == 0 {
curWithoutDeleted[f.Name] = k
}
}
if len(curWithoutDeleted) != len(fs) {
return false
}
for _, f := range fs {
if curWithoutDeleted[f.Name] != keyFor(f) {
return false
}
}
return true
}
func (m *Set) update(cid uint, fs []scanner.File) {
remFiles := m.remoteKey[cid]
for _, f := range fs {
n := f.Name
fk := keyFor(f)
if ck, ok := remFiles[n]; ok && ck == fk {
// The remote already has exactly this file, skip it
continue
}
remFiles[n] = fk
// Keep the block list or increment the usage
if br, ok := m.files[fk]; !ok {
m.files[fk] = fileRecord{
Usage: 1,
File: f,
}
} else {
br.Usage++
m.files[fk] = br
}
// Update global view
gk, ok := m.globalKey[n]
switch {
case ok && fk == gk:
av := m.globalAvailability[n]
av |= 1 << cid
m.globalAvailability[n] = av
case fk.newerThan(gk):
m.globalKey[n] = fk
m.globalAvailability[n] = 1 << cid
}
}
}
func (m *Set) replace(cid uint, fs []scanner.File) {
// Decrement usage for all files belonging to this remote, and remove
// those that are no longer needed.
for _, fk := range m.remoteKey[cid] {
br, ok := m.files[fk]
switch {
case ok && br.Usage == 1:
delete(m.files, fk)
case ok && br.Usage > 1:
br.Usage--
m.files[fk] = br
}
}
// Clear existing remote remoteKey
m.remoteKey[cid] = make(map[string]key)
// Recalculate global based on all remaining remoteKey
for n := range m.globalKey {
var nk key // newest key
var na bitset // newest availability
for i, rem := range m.remoteKey {
if rk, ok := rem[n]; ok {
switch {
case rk == nk:
na |= 1 << uint(i)
case rk.newerThan(nk):
nk = rk
na = 1 << uint(i)
}
}
}
if na != 0 {
// Someone had the file
m.globalKey[n] = nk
m.globalAvailability[n] = na
} else {
// Noone had the file
delete(m.globalKey, n)
delete(m.globalAvailability, n)
}
}
// Add new remote remoteKey to the mix
m.update(cid, fs)
}

321
files/set_test.go Normal file
View File

@@ -0,0 +1,321 @@
package files
import (
"fmt"
"reflect"
"sort"
"testing"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
type fileList []scanner.File
func (l fileList) Len() int {
return len(l)
}
func (l fileList) Less(a, b int) bool {
return l[a].Name < l[b].Name
}
func (l fileList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func TestGlobalSet(t *testing.T) {
m := NewSet()
local := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000},
}
remote := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000},
}
expectedGlobal := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002},
scanner.File{Name: "d", Version: 1000},
scanner.File{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(cid.LocalID, local)
m.Replace(1, remote)
g := m.Global()
sort.Sort(fileList(g))
sort.Sort(fileList(expectedGlobal))
if !reflect.DeepEqual(g, expectedGlobal) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
}
if lb := len(m.files); lb != 7 {
t.Errorf("Num files incorrect %d != 7\n%v", lb, m.files)
}
}
func TestLocalDeleted(t *testing.T) {
m := NewSet()
lamport.Default = lamport.Clock{}
local1 := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000},
}
m.ReplaceWithDelete(cid.LocalID, local1)
local2 := []scanner.File{
local1[0],
local1[2],
}
expectedGlobal1 := []scanner.File{
local1[0],
scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
local1[2],
scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
}
m.ReplaceWithDelete(cid.LocalID, local2)
g := m.Global()
sort.Sort(fileList(g))
sort.Sort(fileList(expectedGlobal1))
if !reflect.DeepEqual(g, expectedGlobal1) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
}
local3 := []scanner.File{
local1[0],
}
expectedGlobal2 := []scanner.File{
local1[0],
scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
scanner.File{Name: "c", Version: 1003, Flags: protocol.FlagDeleted},
scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
}
m.ReplaceWithDelete(cid.LocalID, local3)
g = m.Global()
sort.Sort(fileList(g))
sort.Sort(fileList(expectedGlobal2))
if !reflect.DeepEqual(g, expectedGlobal2) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal2)
}
}
func BenchmarkSetLocal10k(b *testing.B) {
m := NewSet()
var local []scanner.File
for i := 0; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
var remote []scanner.File
for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
m.Replace(1, remote)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.ReplaceWithDelete(cid.LocalID, local)
}
}
func BenchmarkSetLocal10(b *testing.B) {
m := NewSet()
var local []scanner.File
for i := 0; i < 10; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
var remote []scanner.File
for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
m.Replace(1, remote)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.ReplaceWithDelete(cid.LocalID, local)
}
}
func BenchmarkAddLocal10k(b *testing.B) {
m := NewSet()
var local []scanner.File
for i := 0; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
var remote []scanner.File
for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
m.Replace(1, remote)
m.ReplaceWithDelete(cid.LocalID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
for j := range local {
local[j].Version++
}
b.StartTimer()
m.Update(cid.LocalID, local)
}
}
func BenchmarkAddLocal10(b *testing.B) {
m := NewSet()
var local []scanner.File
for i := 0; i < 10; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
var remote []scanner.File
for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
}
m.Replace(1, remote)
m.ReplaceWithDelete(cid.LocalID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for j := range local {
local[j].Version++
}
m.Update(cid.LocalID, local)
}
}
func TestGlobalReset(t *testing.T) {
m := NewSet()
local := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000},
}
remote := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000},
}
expectedGlobalKey := map[string]key{
"a": keyFor(local[0]),
"b": keyFor(local[1]),
"c": keyFor(local[2]),
"d": keyFor(local[3]),
}
m.ReplaceWithDelete(cid.LocalID, local)
m.Replace(1, remote)
m.Replace(1, nil)
if !reflect.DeepEqual(m.globalKey, expectedGlobalKey) {
t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobalKey)
}
if lb := len(m.files); lb != 4 {
t.Errorf("Num files incorrect %d != 4\n%v", lb, m.files)
}
}
func TestNeed(t *testing.T) {
m := NewSet()
local := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000},
}
remote := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000},
}
shouldNeed := []scanner.File{
scanner.File{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(cid.LocalID, local)
m.Replace(1, remote)
need := m.Need(0)
if !reflect.DeepEqual(need, shouldNeed) {
t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
}
}
func TestChanges(t *testing.T) {
m := NewSet()
local1 := []scanner.File{
scanner.File{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000},
}
local2 := []scanner.File{
local1[0],
// [1] deleted
local1[2],
scanner.File{Name: "d", Version: 1002},
scanner.File{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(cid.LocalID, local1)
c0 := m.Changes(cid.LocalID)
m.ReplaceWithDelete(cid.LocalID, local2)
c1 := m.Changes(cid.LocalID)
if !(c1 > c0) {
t.Fatal("Change number should have incremented")
}
m.ReplaceWithDelete(cid.LocalID, local2)
c2 := m.Changes(cid.LocalID)
if c2 != c1 {
t.Fatal("Change number should be unchanged")
}
}