perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The Great Rewrite (fixes #36, #61, #94, #101)

Browse Source 
Rewrite of the file model and pulling mechanism. Needs lots of cleanup
and bugfixes, now...
This commit is contained in:
Jakob Borg
2014-03-28 14:36:57 +01:00
parent 3700eb1e61
commit f87b1520e8
47 changed files with 2137 additions and 1902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
cmd/syncthing/blockqueue.go Normal file
View File

@@ -0,0 +1,94 @@
package main
import "github.com/calmh/syncthing/scanner"
type bqAdd struct {
file scanner.File
have []scanner.Block
need []scanner.Block
}
type bqBlock struct {
file scanner.File
block scanner.Block // get this block from the network
copy []scanner.Block // copy these blocks from the old version of the file
last bool
}
type blockQueue struct {
inbox chan bqAdd
outbox chan bqBlock
queued []bqBlock
}
func newBlockQueue() *blockQueue {
q := &blockQueue{
inbox: make(chan bqAdd),
outbox: make(chan bqBlock),
}
go q.run()
return q
}
func (q *blockQueue) addBlock(a bqAdd) {
// If we already have it queued, return
for _, b := range q.queued {
if b.file.Name == a.file.Name {
return
}
}
if len(a.have) > 0 {
// First queue a copy operation
q.queued = append(q.queued, bqBlock{
file: a.file,
copy: a.have,
})
}
// Queue the needed blocks individually
l := len(a.need)
for i, b := range a.need {
q.queued = append(q.queued, bqBlock{
file: a.file,
block: b,
last: i == l-1,
})
}
if l == 0 {
// If we didn't have anything to fetch, queue an empty block with the "last" flag set to close the file.
q.queued = append(q.queued, bqBlock{
file: a.file,
last: true,
})
}
}
func (q *blockQueue) run() {
for {
if len(q.queued) == 0 {
q.addBlock(<-q.inbox)
} else {
next := q.queued[0]
select {
case a := <-q.inbox:
q.addBlock(a)
case q.outbox <- next:
q.queued = q.queued[1:]
}
}
}
}
func (q *blockQueue) put(a bqAdd) {
q.inbox <- a
}
func (q *blockQueue) get() bqBlock {
return <-q.outbox
}
func (q *blockQueue) empty() bool {
// There is a race condition here. We're only mostly sure the queue is empty if the expression below is true.
return len(q.queued) == 0 && len(q.inbox) == 0 && len(q.outbox) == 0
}

1
cmd/syncthing/config.go
View File

@@ -32,7 +32,6 @@ type NodeConfiguration struct {
type OptionsConfiguration struct {
ListenAddress []string `xml:"listenAddress" default:":22000" ini:"listen-address"`
ReadOnly bool `xml:"readOnly" ini:"read-only"`
AllowDelete bool `xml:"allowDelete" default:"true" ini:"allow-delete"`
FollowSymlinks bool `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
GUIEnabled bool `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
GUIAddress string `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`

2
cmd/syncthing/config_test.go
View File

@@ -11,7 +11,6 @@ func TestDefaultValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{":22000"},
ReadOnly: false,
AllowDelete: true,
FollowSymlinks: true,
GUIEnabled: true,
GUIAddress: "127.0.0.1:8080",
@@ -90,7 +89,6 @@ func TestOverriddenValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{":23000"},
ReadOnly: true,
AllowDelete: false,
FollowSymlinks: false,
GUIEnabled: false,
GUIAddress: "125.2.2.2:8080",

173
cmd/syncthing/filemonitor.go
View File

@@ -1,173 +0,0 @@
package main
import (
"bytes"
"errors"
"fmt"
"os"
"path"
"sync"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/scanner"
)
type fileMonitor struct {
name string // in-repo name
path string // full path
writeDone sync.WaitGroup
model *Model
global scanner.File
localBlocks []scanner.Block
copyError error
writeError error
}
func (m *fileMonitor) FileBegins(cc <-chan content) error {
if debugPull {
dlog.Println("file begins:", m.name)
}
tmp := defTempNamer.TempName(m.path)
dir := path.Dir(tmp)
_, err := os.Stat(dir)
if err != nil && os.IsNotExist(err) {
err = os.MkdirAll(dir, 0777)
if err != nil {
return err
}
}
outFile, err := os.Create(tmp)
if err != nil {
return err
}
m.writeDone.Add(1)
var writeWg sync.WaitGroup
if len(m.localBlocks) > 0 {
writeWg.Add(1)
inFile, err := os.Open(m.path)
if err != nil {
return err
}
// Copy local blocks, close infile when done
go m.copyLocalBlocks(inFile, outFile, &writeWg)
}
// Write remote blocks,
writeWg.Add(1)
go m.copyRemoteBlocks(cc, outFile, &writeWg)
// Wait for both writing routines, then close the outfile
go func() {
writeWg.Wait()
outFile.Close()
m.writeDone.Done()
}()
return nil
}
func (m *fileMonitor) copyLocalBlocks(inFile, outFile *os.File, writeWg *sync.WaitGroup) {
defer inFile.Close()
defer writeWg.Done()
var buf = buffers.Get(BlockSize)
defer buffers.Put(buf)
for _, lb := range m.localBlocks {
buf = buf[:lb.Size]
_, err := inFile.ReadAt(buf, lb.Offset)
if err != nil {
m.copyError = err
return
}
_, err = outFile.WriteAt(buf, lb.Offset)
if err != nil {
m.copyError = err
return
}
}
}
func (m *fileMonitor) copyRemoteBlocks(cc <-chan content, outFile *os.File, writeWg *sync.WaitGroup) {
defer writeWg.Done()
for content := range cc {
_, err := outFile.WriteAt(content.data, content.offset)
buffers.Put(content.data)
if err != nil {
m.writeError = err
return
}
}
}
func (m *fileMonitor) FileDone() error {
if debugPull {
dlog.Println("file done:", m.name)
}
m.writeDone.Wait()
tmp := defTempNamer.TempName(m.path)
defer os.Remove(tmp)
if m.copyError != nil {
return m.copyError
}
if m.writeError != nil {
return m.writeError
}
err := hashCheck(tmp, m.global.Blocks)
if err != nil {
return err
}
err = os.Chtimes(tmp, time.Unix(m.global.Modified, 0), time.Unix(m.global.Modified, 0))
if err != nil {
return err
}
err = os.Chmod(tmp, os.FileMode(m.global.Flags&0777))
if err != nil {
return err
}
err = os.Rename(tmp, m.path)
if err != nil {
return err
}
m.model.updateLocal(m.global)
return nil
}
func hashCheck(name string, correct []scanner.Block) error {
rf, err := os.Open(name)
if err != nil {
return err
}
defer rf.Close()
current, err := scanner.Blocks(rf, BlockSize)
if err != nil {
return err
}
if len(current) != len(correct) {
return errors.New("incorrect number of blocks")
}
for i := range current {
if bytes.Compare(current[i].Hash, correct[i].Hash) != 0 {
return fmt.Errorf("hash mismatch: %x != %x", current[i], correct[i])
}
}
return nil
}

241
cmd/syncthing/filequeue.go
View File

@@ -1,241 +0,0 @@
package main
import (
"log"
"sort"
"sync"
"time"
"github.com/calmh/syncthing/scanner"
)
type Monitor interface {
FileBegins(<-chan content) error
FileDone() error
}
type FileQueue struct {
files queuedFileList
sorted bool
fmut sync.Mutex // protects files and sorted
availability map[string][]string
amut sync.Mutex // protects availability
queued map[string]bool
}
type queuedFile struct {
name string
blocks []scanner.Block
activeBlocks []bool
given int
remaining int
channel chan content
nodes []string
nodesChecked time.Time
monitor Monitor
}
type content struct {
offset int64
data []byte
}
type queuedFileList []queuedFile
func (l queuedFileList) Len() int { return len(l) }
func (l queuedFileList) Swap(a, b int) { l[a], l[b] = l[b], l[a] }
func (l queuedFileList) Less(a, b int) bool {
// Sort by most blocks already given out, then alphabetically
if l[a].given != l[b].given {
return l[a].given > l[b].given
}
return l[a].name < l[b].name
}
type queuedBlock struct {
name string
block scanner.Block
index int
}
func NewFileQueue() *FileQueue {
return &FileQueue{
availability: make(map[string][]string),
queued: make(map[string]bool),
}
}
func (q *FileQueue) Add(name string, blocks []scanner.Block, monitor Monitor) {
q.fmut.Lock()
defer q.fmut.Unlock()
if q.queued[name] {
return
}
q.files = append(q.files, queuedFile{
name: name,
blocks: blocks,
activeBlocks: make([]bool, len(blocks)),
remaining: len(blocks),
channel: make(chan content),
monitor: monitor,
})
q.queued[name] = true
q.sorted = false
}
func (q *FileQueue) Len() int {
q.fmut.Lock()
defer q.fmut.Unlock()
return len(q.files)
}
func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
q.fmut.Lock()
defer q.fmut.Unlock()
if !q.sorted {
sort.Sort(q.files)
q.sorted = true
}
for i := range q.files {
qf := &q.files[i]
q.amut.Lock()
av := q.availability[qf.name]
q.amut.Unlock()
if len(av) == 0 {
// Noone has the file we want; abort.
if qf.remaining != len(qf.blocks) {
// We have already started on this file; close it down
close(qf.channel)
if mon := qf.monitor; mon != nil {
mon.FileDone()
}
}
delete(q.queued, qf.name)
q.deleteAt(i)
return queuedBlock{}, false
}
for _, ni := range av {
// Find and return the next block in the queue
if ni == nodeID {
for j, b := range qf.blocks {
if !qf.activeBlocks[j] {
qf.activeBlocks[j] = true
qf.given++
return queuedBlock{
name: qf.name,
block: b,
index: j,
}, true
}
}
break
}
}
}
// We found nothing to do
return queuedBlock{}, false
}
func (q *FileQueue) Done(file string, offset int64, data []byte) {
q.fmut.Lock()
defer q.fmut.Unlock()
c := content{
offset: offset,
data: data,
}
for i := range q.files {
qf := &q.files[i]
if qf.name == file {
if qf.monitor != nil && qf.remaining == len(qf.blocks) {
err := qf.monitor.FileBegins(qf.channel)
if err != nil {
log.Printf("WARNING: %s: %v (not synced)", qf.name, err)
delete(q.queued, qf.name)
q.deleteAt(i)
return
}
}
qf.channel <- c
qf.remaining--
if qf.remaining == 0 {
close(qf.channel)
if qf.monitor != nil {
err := qf.monitor.FileDone()
if err != nil {
log.Printf("WARNING: %s: %v", qf.name, err)
}
}
delete(q.queued, qf.name)
q.deleteAt(i)
}
return
}
}
// We found nothing, might have errored out already
}
func (q *FileQueue) QueuedFiles() (files []string) {
q.fmut.Lock()
defer q.fmut.Unlock()
for _, qf := range q.files {
files = append(files, qf.name)
}
return
}
func (q *FileQueue) deleteAt(i int) {
q.files = append(q.files[:i], q.files[i+1:]...)
}
func (q *FileQueue) deleteFile(n string) {
for i, file := range q.files {
if n == file.name {
q.deleteAt(i)
delete(q.queued, file.name)
return
}
}
}
func (q *FileQueue) SetAvailable(file string, nodes []string) {
q.amut.Lock()
defer q.amut.Unlock()
q.availability[file] = nodes
}
func (q *FileQueue) RemoveAvailable(toRemove string) {
q.fmut.Lock()
q.amut.Lock()
defer q.amut.Unlock()
defer q.fmut.Unlock()
for file, nodes := range q.availability {
for i, node := range nodes {
if node == toRemove {
q.availability[file] = nodes[:i+copy(nodes[i:], nodes[i+1:])]
if len(q.availability[file]) == 0 {
q.deleteFile(file)
}
}
break
}
}
}

297
cmd/syncthing/filequeue_test.go
View File

@@ -1,297 +0,0 @@
package main
import (
"reflect"
"sync"
"sync/atomic"
"testing"
"github.com/calmh/syncthing/scanner"
)
func TestFileQueueAdd(t *testing.T) {
q := NewFileQueue()
q.Add("foo", nil, nil)
}
func TestFileQueueAddSorting(t *testing.T) {
q := NewFileQueue()
q.SetAvailable("zzz", []string{"nodeID"})
q.SetAvailable("aaa", []string{"nodeID"})
q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ := q.Get("nodeID")
if b.name != "aaa" {
t.Errorf("Incorrectly sorted get: %+v", b)
}
q = NewFileQueue()
q.SetAvailable("zzz", []string{"nodeID"})
q.SetAvailable("aaa", []string{"nodeID"})
q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ = q.Get("nodeID") // Start on zzzz
if b.name != "zzz" {
t.Errorf("Incorrectly sorted get: %+v", b)
}
q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ = q.Get("nodeID")
if b.name != "zzz" {
// Continue rather than starting a new file
t.Errorf("Incorrectly sorted get: %+v", b)
}
}
func TestFileQueueLen(t *testing.T) {
q := NewFileQueue()
q.Add("foo", nil, nil)
q.Add("bar", nil, nil)
if l := q.Len(); l != 2 {
t.Errorf("Incorrect len %d != 2 after adds", l)
}
}
func TestFileQueueGet(t *testing.T) {
q := NewFileQueue()
q.SetAvailable("foo", []string{"nodeID"})
q.SetAvailable("bar", []string{"nodeID"})
q.Add("foo", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
}, nil)
q.Add("bar", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
}, nil)
// First get should return the first block of the first file
expected := queuedBlock{
name: "bar",
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some bar hash bytes"),
},
}
actual, ok := q.Get("nodeID")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned (first)\n E: %+v\n A: %+v", expected, actual)
}
// Second get should return the next block of the first file
expected = queuedBlock{
name: "bar",
block: scanner.Block{
Offset: 128,
Size: 128,
Hash: []byte("some other bar hash bytes"),
},
index: 1,
}
actual, ok = q.Get("nodeID")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned (second)\n E: %+v\n A: %+v", expected, actual)
}
// Third get should return the first block of the second file
expected = queuedBlock{
name: "foo",
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some foo hash bytes"),
},
}
actual, ok = q.Get("nodeID")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned (third)\n E: %+v\n A: %+v", expected, actual)
}
}
/*
func TestFileQueueDone(t *testing.T) {
ch := make(chan content)
var recv sync.WaitGroup
recv.Add(1)
go func() {
content := <-ch
if bytes.Compare(content.data, []byte("first block bytes")) != 0 {
t.Error("Incorrect data in first content block")
}
content = <-ch
if bytes.Compare(content.data, []byte("second block bytes")) != 0 {
t.Error("Incorrect data in second content block")
}
_, ok := <-ch
if ok {
t.Error("Content channel not closed")
}
recv.Done()
}()
q := FileQueue{resolver: fakeResolver{}}
q.Add("foo", []scanner.Block{
{Offset: 0, Length: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Length: 128, Hash: []byte("some other foo hash bytes")},
}, ch)
b0, _ := q.Get("nodeID")
b1, _ := q.Get("nodeID")
q.Done(b0.name, b0.block.Offset, []byte("first block bytes"))
q.Done(b1.name, b1.block.Offset, []byte("second block bytes"))
recv.Wait()
// Queue should now have one file less
if l := q.Len(); l != 0 {
t.Error("Queue not empty")
}
_, ok := q.Get("nodeID")
if ok {
t.Error("Unexpected OK Get()")
}
}
*/
func TestFileQueueGetNodeIDs(t *testing.T) {
q := NewFileQueue()
q.SetAvailable("a-foo", []string{"nodeID", "a"})
q.SetAvailable("b-bar", []string{"nodeID", "b"})
q.Add("a-foo", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
}, nil)
q.Add("b-bar", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
}, nil)
expected := queuedBlock{
name: "b-bar",
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some bar hash bytes"),
},
}
actual, ok := q.Get("b")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned\n E: %+v\n A: %+v", expected, actual)
}
expected = queuedBlock{
name: "a-foo",
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some foo hash bytes"),
},
}
actual, ok = q.Get("a")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned\n E: %+v\n A: %+v", expected, actual)
}
expected = queuedBlock{
name: "a-foo",
block: scanner.Block{
Offset: 128,
Size: 128,
Hash: []byte("some other foo hash bytes"),
},
index: 1,
}
actual, ok = q.Get("nodeID")
if !ok {
t.Error("Unexpected non-OK Get()")
}
if !reflect.DeepEqual(expected, actual) {
t.Errorf("Incorrect block returned\n E: %+v\n A: %+v", expected, actual)
}
}
func TestFileQueueThreadHandling(t *testing.T) {
// This should pass with go test -race
const n = 100
var total int
var blocks []scanner.Block
for i := 1; i <= n; i++ {
blocks = append(blocks, scanner.Block{Offset: int64(i), Size: 1})
total += i
}
q := NewFileQueue()
q.Add("foo", blocks, nil)
q.SetAvailable("foo", []string{"nodeID"})
var start = make(chan bool)
var gotTot uint32
var wg sync.WaitGroup
wg.Add(n)
for i := 1; i <= n; i++ {
go func() {
<-start
b, _ := q.Get("nodeID")
atomic.AddUint32(&gotTot, uint32(b.block.Offset))
wg.Done()
}()
}
close(start)
wg.Wait()
if int(gotTot) != total {
t.Errorf("Total mismatch; %d != %d", gotTot, total)
}
}
func TestDeleteAt(t *testing.T) {
q := FileQueue{}
for i := 0; i < 4; i++ {
q.files = queuedFileList{{name: "a"}, {name: "b"}, {name: "c"}, {name: "d"}}
q.deleteAt(i)
if l := len(q.files); l != 3 {
t.Fatalf("deleteAt(%d) failed; %d != 3", i, l)
}
}
q.files = queuedFileList{{name: "a"}}
q.deleteAt(0)
if l := len(q.files); l != 0 {
t.Fatalf("deleteAt(only) failed; %d != 0", l)
}
}

101
cmd/syncthing/main.go
View File

@@ -11,7 +11,7 @@ import (
_ "net/http/pprof"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
@@ -51,12 +51,15 @@ const (
STTRACE A comma separated string of facilities to trace. The valid
facility strings:
- "scanner" (the file change scanner)
- "discover" (the node discovery package)
- "net" (connecting and disconnecting, network messages)
- "files" (file set store)
- "idx" (index sending and receiving)
- "mc" (multicast beacon)
- "need" (file need calculations)
- "pull" (file pull activity)`
- "net" (connecting and disconnecting, network messages)
- "pull" (file pull activity)
- "scanner" (the file change scanner)
`
)
func main() {
@@ -105,7 +108,7 @@ func main() {
// Prepare to be able to save configuration
cfgFile := path.Join(confDir, "config.xml")
cfgFile := filepath.Join(confDir, "config.xml")
go saveConfigLoop(cfgFile)
// Load the configuration file, if it exists.
@@ -121,13 +124,13 @@ func main() {
cf.Close()
} else {
// No config.xml, let's try the old syncthing.ini
iniFile := path.Join(confDir, "syncthing.ini")
iniFile := filepath.Join(confDir, "syncthing.ini")
cf, err := os.Open(iniFile)
if err == nil {
infoln("Migrating syncthing.ini to config.xml")
iniCfg := ini.Parse(cf)
cf.Close()
os.Rename(iniFile, path.Join(confDir, "migrated_syncthing.ini"))
Rename(iniFile, filepath.Join(confDir, "migrated_syncthing.ini"))
cfg, _ = readConfigXML(nil)
cfg.Repositories = []RepositoryConfiguration{
@@ -152,7 +155,7 @@ func main() {
cfg, err = readConfigXML(nil)
cfg.Repositories = []RepositoryConfiguration{
{
Directory: path.Join(getHomeDir(), "Sync"),
Directory: filepath.Join(getHomeDir(), "Sync"),
Nodes: []NodeConfiguration{
{NodeID: myID, Addresses: []string{"dynamic"}},
},
@@ -259,35 +262,16 @@ func main() {
// Routine to pull blocks from other nodes to synchronize the local
// repository. Does not run when we are in read only (publish only) mode.
if !cfg.Options.ReadOnly {
if cfg.Options.ReadOnly {
if verbose {
okln("Ready to synchronize (read only; no external updates accepted)")
}
m.StartRO()
} else {
if verbose {
if cfg.Options.AllowDelete {
infoln("Deletes from peer nodes are allowed")
} else {
infoln("Deletes from peer nodes will be ignored")
}
okln("Ready to synchronize (read-write)")
}
m.StartRW(cfg.Options.AllowDelete, cfg.Options.ParallelRequests)
} else if verbose {
okln("Ready to synchronize (read only; no external updates accepted)")
}
// Periodically scan the repository and update the local
// XXX: Should use some fsnotify mechanism.
go func() {
td := time.Duration(cfg.Options.RescanIntervalS) * time.Second
for {
time.Sleep(td)
if m.LocalAge() > (td / 2).Seconds() {
updateLocalModel(m, w)
}
}
}()
if verbose {
// Periodically print statistics
go printStatsLoop(m)
m.StartRW(cfg.Options.ParallelRequests)
}
select {}
@@ -344,14 +328,7 @@ func saveConfigLoop(cfgFile string) {
continue
}
if runtime.GOOS == "windows" {
err := os.Remove(cfgFile)
if err != nil && !os.IsNotExist(err) {
warnln(err)
}
}
err = os.Rename(cfgFile+".tmp", cfgFile)
err = Rename(cfgFile+".tmp", cfgFile)
if err != nil {
warnln(err)
}
@@ -362,37 +339,6 @@ func saveConfig() {
saveConfigCh <- struct{}{}
}
func printStatsLoop(m *Model) {
var lastUpdated int64
var lastStats = make(map[string]ConnectionInfo)
for {
time.Sleep(60 * time.Second)
for node, stats := range m.ConnectionStats() {
secs := time.Since(lastStats[node].At).Seconds()
inbps := 8 * int(float64(stats.InBytesTotal-lastStats[node].InBytesTotal)/secs)
outbps := 8 * int(float64(stats.OutBytesTotal-lastStats[node].OutBytesTotal)/secs)
if inbps+outbps > 0 {
infof("%s: %sb/s in, %sb/s out", node[0:5], MetricPrefix(int64(inbps)), MetricPrefix(int64(outbps)))
}
lastStats[node] = stats
}
if lu := m.Generation(); lu > lastUpdated {
lastUpdated = lu
files, _, bytes := m.GlobalSize()
infof("%6d files, %9sB in cluster", files, BinaryPrefix(bytes))
files, _, bytes = m.LocalSize()
infof("%6d files, %9sB in local repo", files, BinaryPrefix(bytes))
needFiles, bytes := m.NeedFiles()
infof("%6d files, %9sB to synchronize", len(needFiles), BinaryPrefix(bytes))
}
}
}
func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
var conns = make(chan *tls.Conn)
@@ -529,7 +475,7 @@ func updateLocalModel(m *Model, w *scanner.Walker) {
func saveIndex(m *Model) {
name := m.RepoID() + ".idx.gz"
fullName := path.Join(confDir, name)
fullName := filepath.Join(confDir, name)
idxf, err := os.Create(fullName + ".tmp")
if err != nil {
return
@@ -543,12 +489,13 @@ func saveIndex(m *Model) {
}.EncodeXDR(gzw)
gzw.Close()
idxf.Close()
os.Rename(fullName+".tmp", fullName)
Rename(fullName+".tmp", fullName)
}
func loadIndex(m *Model) {
name := m.RepoID() + ".idx.gz"
idxf, err := os.Open(path.Join(confDir, name))
idxf, err := os.Open(filepath.Join(confDir, name))
if err != nil {
return
}
@@ -611,7 +558,7 @@ func getHomeDir() string {
func getDefaultConfDir() string {
if runtime.GOOS == "windows" {
return path.Join(os.Getenv("AppData"), "syncthing")
return filepath.Join(os.Getenv("AppData"), "syncthing")
}
return expandTilde("~/.syncthing")
}

684
cmd/syncthing/model.go
View File

@@ -7,59 +7,36 @@ import (
"io"
"net"
"os"
"path"
"path/filepath"
"sync"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
type Model struct {
dir string
cm *cid.Map
fs *files.Set
global map[string]scanner.File // the latest version of each file as it exists in the cluster
gmut sync.RWMutex // protects global
local map[string]scanner.File // the files we currently have locally on disk
lmut sync.RWMutex // protects local
remote map[string]map[string]scanner.File
rmut sync.RWMutex // protects remote
protoConn map[string]Connection
protoConn map[string]protocol.Connection
rawConn map[string]io.Closer
pmut sync.RWMutex // protects protoConn and rawConn
// Queue for files to fetch. fq can call back into the model, so we must ensure
// to hold no locks when calling methods on fq.
fq *FileQueue
dq chan scanner.File // queue for files to delete
updatedLocal int64 // timestamp of last update to local
updateGlobal int64 // timestamp of last update to remote
lastIdxBcast time.Time
lastIdxBcastRequest time.Time
umut sync.RWMutex // provides updated* and lastIdx*
rwRunning bool
delete bool
initmut sync.Mutex // protects rwRunning and delete
initOnce sync.Once
sup suppressor
parallelRequests int
limitRequestRate chan struct{}
imut sync.Mutex // protects Index
}
type Connection interface {
ID() string
Index(string, []protocol.FileInfo)
Request(repo, name string, offset int64, size int) ([]byte, error)
Statistics() protocol.Statistics
Option(key string) string
}
const (
idxBcastHoldtime = 15 * time.Second // Wait at least this long after the last index modification
idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
@@ -75,16 +52,12 @@ var (
// for file data without altering the local repository in any way.
func NewModel(dir string, maxChangeBw int) *Model {
m := &Model{
dir: dir,
global: make(map[string]scanner.File),
local: make(map[string]scanner.File),
remote: make(map[string]map[string]scanner.File),
protoConn: make(map[string]Connection),
rawConn: make(map[string]io.Closer),
lastIdxBcast: time.Now(),
sup: suppressor{threshold: int64(maxChangeBw)},
fq: NewFileQueue(),
dq: make(chan scanner.File),
dir: dir,
cm: cid.NewMap(),
fs: files.NewSet(),
protoConn: make(map[string]protocol.Connection),
rawConn: make(map[string]io.Closer),
sup: suppressor{threshold: int64(maxChangeBw)},
}
go m.broadcastIndexLoop()
@@ -109,37 +82,26 @@ func (m *Model) LimitRate(kbps int) {
// StartRW starts read/write processing on the current model. When in
// read/write mode the model will attempt to keep in sync with the cluster by
// pulling needed files from peer nodes.
func (m *Model) StartRW(del bool, threads int) {
m.initmut.Lock()
defer m.initmut.Unlock()
func (m *Model) StartRW(threads int) {
m.initOnce.Do(func() {
newPuller("default", m.dir, m, threads)
})
}
if m.rwRunning {
panic("starting started model")
}
m.rwRunning = true
m.delete = del
m.parallelRequests = threads
if del {
go m.deleteLoop()
}
// StartRO starts read only processing on the current model. When in
// read only mode the model will announce files to the cluster but not
// pull in any external changes.
func (m *Model) StartRO() {
m.initOnce.Do(func() {
newPuller("default", m.dir, m, 0) // zero threads => read only
})
}
// Generation returns an opaque integer that is guaranteed to increment on
// every change to the local repository or global model.
func (m *Model) Generation() int64 {
m.umut.RLock()
defer m.umut.RUnlock()
return m.updatedLocal + m.updateGlobal
}
func (m *Model) LocalAge() float64 {
m.umut.RLock()
defer m.umut.RUnlock()
return time.Since(time.Unix(m.updatedLocal, 0)).Seconds()
func (m *Model) Generation() uint64 {
c := m.fs.Changes(cid.LocalID)
return c
}
type ConnectionInfo struct {
@@ -156,12 +118,10 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
RemoteAddr() net.Addr
}
m.gmut.RLock()
m.pmut.RLock()
m.rmut.RLock()
var tot int64
for _, f := range m.global {
for _, f := range m.fs.Global() {
if f.Flags&protocol.FlagDeleted == 0 {
tot += f.Size
}
@@ -178,10 +138,10 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
ci.Address = nc.RemoteAddr().String()
}
var have int64
for _, f := range m.remote[node] {
if f.Equals(m.global[f.Name]) && f.Flags&protocol.FlagDeleted == 0 {
have += f.Size
var have = tot
for _, f := range m.fs.Need(m.cm.Get(node)) {
if f.Flags&protocol.FlagDeleted == 0 {
have -= f.Size
}
}
@@ -193,82 +153,59 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
res[node] = ci
}
m.rmut.RUnlock()
m.pmut.RUnlock()
m.gmut.RUnlock()
return res
}
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
for _, f := range fs {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size
} else {
deleted++
}
}
return
}
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize() (files, deleted int, bytes int64) {
m.gmut.RLock()
for _, f := range m.global {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size
} else {
deleted++
}
}
m.gmut.RUnlock()
return
fs := m.fs.Global()
return sizeOf(fs)
}
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local repository.
func (m *Model) LocalSize() (files, deleted int, bytes int64) {
m.lmut.RLock()
for _, f := range m.local {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size
} else {
deleted++
}
}
m.lmut.RUnlock()
return
fs := m.fs.Have(cid.LocalID)
return sizeOf(fs)
}
// InSyncSize returns the number and total byte size of the local files that
// are in sync with the global model.
func (m *Model) InSyncSize() (files, bytes int64) {
m.gmut.RLock()
m.lmut.RLock()
func (m *Model) InSyncSize() (files int, bytes int64) {
gf := m.fs.Global()
hf := m.fs.Need(cid.LocalID)
for n, f := range m.local {
if gf, ok := m.global[n]; ok && f.Equals(gf) {
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size
}
}
}
gn, _, gb := sizeOf(gf)
hn, _, hb := sizeOf(hf)
m.lmut.RUnlock()
m.gmut.RUnlock()
return
return gn - hn, gb - hb
}
// NeedFiles returns the list of currently needed files and the total size.
func (m *Model) NeedFiles() (files []scanner.File, bytes int64) {
qf := m.fq.QueuedFiles()
func (m *Model) NeedFiles() ([]scanner.File, int64) {
nf := m.fs.Need(cid.LocalID)
m.gmut.RLock()
for _, n := range qf {
f := m.global[n]
files = append(files, f)
var bytes int64
for _, f := range nf {
bytes += f.Size
}
m.gmut.RUnlock()
return
return nf, bytes
}
// Index is called when a new node is connected and we receive their full index.
@@ -276,27 +213,16 @@ func (m *Model) NeedFiles() (files []scanner.File, bytes int64) {
func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
var files = make([]scanner.File, len(fs))
for i := range fs {
lamport.Default.Tick(fs[i].Version)
files[i] = fileFromFileInfo(fs[i])
}
m.imut.Lock()
defer m.imut.Unlock()
cid := m.cm.Get(nodeID)
m.fs.Replace(cid, files)
if debugNet {
dlog.Printf("IDX(in): %s: %d files", nodeID, len(fs))
}
repo := make(map[string]scanner.File)
for _, f := range files {
m.indexUpdate(repo, f)
}
m.rmut.Lock()
m.remote[nodeID] = repo
m.rmut.Unlock()
m.recomputeGlobal()
m.recomputeNeedForFiles(files)
}
// IndexUpdate is called for incremental updates to connected nodes' indexes.
@@ -304,48 +230,16 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
var files = make([]scanner.File, len(fs))
for i := range fs {
lamport.Default.Tick(fs[i].Version)
files[i] = fileFromFileInfo(fs[i])
}
m.imut.Lock()
defer m.imut.Unlock()
id := m.cm.Get(nodeID)
m.fs.Update(id, files)
if debugNet {
dlog.Printf("IDXUP(in): %s: %d files", nodeID, len(files))
}
m.rmut.Lock()
repo, ok := m.remote[nodeID]
if !ok {
warnf("Index update from node %s that does not have an index", nodeID)
m.rmut.Unlock()
return
}
for _, f := range files {
m.indexUpdate(repo, f)
}
m.rmut.Unlock()
m.recomputeGlobal()
m.recomputeNeedForFiles(files)
}
func (m *Model) indexUpdate(repo map[string]scanner.File, f scanner.File) {
if debugIdx {
var flagComment string
if f.Flags&protocol.FlagDeleted != 0 {
flagComment = " (deleted)"
}
dlog.Printf("IDX(in): %q m=%d f=%o%s v=%d (%d blocks)", f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
if extraFlags := f.Flags &^ (protocol.FlagInvalid | protocol.FlagDeleted | 0xfff); extraFlags != 0 {
warnf("IDX(in): Unknown flags 0x%x in index record %+v", extraFlags, f)
return
}
repo[f.Name] = f
}
// Close removes the peer from the model and closes the underlying connection if possible.
@@ -360,40 +254,26 @@ func (m *Model) Close(node string, err error) {
warnf("Connection to %s closed: %v", node, err)
}
m.fq.RemoveAvailable(node)
cid := m.cm.Get(node)
m.fs.Replace(cid, nil)
m.cm.Clear(node)
m.pmut.Lock()
m.rmut.Lock()
conn, ok := m.rawConn[node]
if ok {
conn.Close()
}
delete(m.remote, node)
delete(m.protoConn, node)
delete(m.rawConn, node)
m.rmut.Unlock()
m.pmut.Unlock()
m.recomputeGlobal()
m.recomputeNeedForGlobal()
}
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]byte, error) {
// Verify that the requested file exists in the local and global model.
m.lmut.RLock()
lf, localOk := m.local[name]
m.lmut.RUnlock()
m.gmut.RLock()
_, globalOk := m.global[name]
m.gmut.RUnlock()
if !localOk || !globalOk {
// Verify that the requested file exists in the local model.
lf := m.fs.Get(cid.LocalID, name)
if offset > lf.Size {
warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
return nil, ErrNoSuchFile
}
@@ -404,7 +284,7 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
if debugNet && nodeID != "<local>" {
dlog.Printf("REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
}
fn := path.Join(m.dir, name)
fn := filepath.Join(m.dir, name)
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
if err != nil {
return nil, err
@@ -428,63 +308,23 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
// ReplaceLocal replaces the local repository index with the given list of files.
func (m *Model) ReplaceLocal(fs []scanner.File) {
var updated bool
var newLocal = make(map[string]scanner.File)
m.lmut.RLock()
for _, f := range fs {
newLocal[f.Name] = f
if ef := m.local[f.Name]; !ef.Equals(f) {
updated = true
}
}
m.lmut.RUnlock()
if m.markDeletedLocals(newLocal) {
updated = true
}
m.lmut.RLock()
if len(newLocal) != len(m.local) {
updated = true
}
m.lmut.RUnlock()
if updated {
m.lmut.Lock()
m.local = newLocal
m.lmut.Unlock()
m.recomputeGlobal()
m.recomputeNeedForGlobal()
m.umut.Lock()
m.updatedLocal = time.Now().Unix()
m.lastIdxBcastRequest = time.Now()
m.umut.Unlock()
}
m.fs.ReplaceWithDelete(cid.LocalID, fs)
}
// SeedLocal replaces the local repository index with the given list of files,
// in protocol data types. Does not track deletes, should only be used to seed
// the local index from a cache file at startup.
// ReplaceLocal replaces the local repository index with the given list of files.
func (m *Model) SeedLocal(fs []protocol.FileInfo) {
m.lmut.Lock()
m.local = make(map[string]scanner.File)
for _, f := range fs {
m.local[f.Name] = fileFromFileInfo(f)
var sfs = make([]scanner.File, len(fs))
for i := 0; i < len(fs); i++ {
lamport.Default.Tick(fs[i].Version)
sfs[i] = fileFromFileInfo(fs[i])
}
m.lmut.Unlock()
m.recomputeGlobal()
m.recomputeNeedForGlobal()
m.fs.Replace(cid.LocalID, sfs)
}
// Implements scanner.CurrentFiler
func (m *Model) CurrentFile(file string) scanner.File {
m.lmut.RLock()
f := m.local[file]
m.lmut.RUnlock()
f := m.fs.Get(cid.LocalID, file)
return f
}
@@ -504,7 +344,7 @@ func (m *Model) RepoID() string {
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
// repository changes.
func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
nodeID := protoConn.ID()
m.pmut.Lock()
if _, ok := m.protoConn[nodeID]; ok {
@@ -524,44 +364,6 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
}
protoConn.Index("default", idx)
}()
m.initmut.Lock()
rw := m.rwRunning
m.initmut.Unlock()
if !rw {
return
}
for i := 0; i < m.parallelRequests; i++ {
i := i
go func() {
if debugPull {
dlog.Println("starting puller:", nodeID, i)
}
for {
m.pmut.RLock()
if _, ok := m.protoConn[nodeID]; !ok {
if debugPull {
dlog.Println("stopping puller:", nodeID, i)
}
m.pmut.RUnlock()
return
}
m.pmut.RUnlock()
qb, ok := m.fq.Get(nodeID)
if ok {
if debugPull {
dlog.Println("request: out", nodeID, i, qb.name, qb.block.Offset)
}
data, _ := protoConn.Request("default", qb.name, qb.block.Offset, int(qb.block.Size))
m.fq.Done(qb.name, qb.block.Offset, data)
} else {
time.Sleep(1 * time.Second)
}
}
}()
}
}
// ProtocolIndex returns the current local index in protocol data types.
@@ -569,9 +371,9 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
func (m *Model) ProtocolIndex() []protocol.FileInfo {
var index []protocol.FileInfo
m.lmut.RLock()
fs := m.fs.Have(cid.LocalID)
for _, f := range m.local {
for _, f := range fs {
mf := fileInfoFromFile(f)
if debugIdx {
var flagComment string
@@ -583,10 +385,13 @@ func (m *Model) ProtocolIndex() []protocol.FileInfo {
index = append(index, mf)
}
m.lmut.RUnlock()
return index
}
func (m *Model) updateLocal(f scanner.File) {
m.fs.Update(cid.LocalID, []scanner.File{f})
}
func (m *Model) requestGlobal(nodeID, name string, offset int64, size int, hash []byte) ([]byte, error) {
m.pmut.RLock()
nc, ok := m.protoConn[nodeID]
@@ -604,290 +409,42 @@ func (m *Model) requestGlobal(nodeID, name string, offset int64, size int, hash
}
func (m *Model) broadcastIndexLoop() {
var lastChange uint64
for {
m.umut.RLock()
bcastRequested := m.lastIdxBcastRequest.After(m.lastIdxBcast)
holdtimeExceeded := time.Since(m.lastIdxBcastRequest) > idxBcastHoldtime
m.umut.RUnlock()
time.Sleep(5 * time.Second)
maxDelayExceeded := time.Since(m.lastIdxBcast) > idxBcastMaxDelay
if bcastRequested && (holdtimeExceeded || maxDelayExceeded) {
idx := m.ProtocolIndex()
c := m.fs.Changes(cid.LocalID)
if c == lastChange {
continue
}
lastChange = c
var indexWg sync.WaitGroup
indexWg.Add(len(m.protoConn))
saveIndex(m) // This should be cleaned up we don't do a lot of processing twice
m.umut.Lock()
m.lastIdxBcast = time.Now()
m.umut.Unlock()
fs := m.fs.Have(cid.LocalID)
m.pmut.RLock()
for _, node := range m.protoConn {
node := node
if debugNet {
dlog.Printf("IDX(out/loop): %s: %d files", node.ID(), len(idx))
}
go func() {
node.Index("default", idx)
indexWg.Done()
}()
var indexWg sync.WaitGroup
indexWg.Add(len(m.protoConn))
var idx = make([]protocol.FileInfo, len(fs))
for i, f := range fs {
idx[i] = fileInfoFromFile(f)
}
m.pmut.RLock()
for _, node := range m.protoConn {
node := node
if debugNet {
dlog.Printf("IDX(out/loop): %s: %d files", node.ID(), len(idx))
}
m.pmut.RUnlock()
indexWg.Wait()
go func() {
node.Index("default", idx)
indexWg.Done()
}()
}
time.Sleep(idxBcastHoldtime)
}
}
m.pmut.RUnlock()
// markDeletedLocals sets the deleted flag on files that have gone missing locally.
func (m *Model) markDeletedLocals(newLocal map[string]scanner.File) bool {
// For every file in the existing local table, check if they are also
// present in the new local table. If they are not, check that we already
// had the newest version available according to the global table and if so
// note the file as having been deleted.
var updated bool
m.gmut.RLock()
m.lmut.RLock()
for n, f := range m.local {
if _, ok := newLocal[n]; !ok {
if gf := m.global[n]; !gf.NewerThan(f) {
if f.Flags&protocol.FlagDeleted == 0 {
f.Flags = protocol.FlagDeleted
f.Version++
f.Blocks = nil
updated = true
}
newLocal[n] = f
}
}
}
m.lmut.RUnlock()
m.gmut.RUnlock()
return updated
}
func (m *Model) updateLocal(f scanner.File) {
var updated bool
m.lmut.Lock()
if ef, ok := m.local[f.Name]; !ok || !ef.Equals(f) {
m.local[f.Name] = f
updated = true
}
m.lmut.Unlock()
if updated {
m.recomputeGlobal()
// We don't recomputeNeed here for two reasons:
// - a need shouldn't have arisen due to having a newer local file
// - recomputeNeed might call into fq.Add but we might have been called by
// fq which would be a deadlock on fq
m.umut.Lock()
m.updatedLocal = time.Now().Unix()
m.lastIdxBcastRequest = time.Now()
m.umut.Unlock()
}
}
/*
XXX: Not done, needs elegant handling of availability
func (m *Model) recomputeGlobalFor(files []scanner.File) bool {
m.gmut.Lock()
defer m.gmut.Unlock()
var updated bool
for _, f := range files {
if gf, ok := m.global[f.Name]; !ok || f.NewerThan(gf) {
m.global[f.Name] = f
updated = true
// Fix availability
}
}
return updated
}
*/
func (m *Model) recomputeGlobal() {
var newGlobal = make(map[string]scanner.File)
m.lmut.RLock()
for n, f := range m.local {
newGlobal[n] = f
}
m.lmut.RUnlock()
var available = make(map[string][]string)
m.rmut.RLock()
var highestMod int64
for nodeID, fs := range m.remote {
for n, nf := range fs {
if lf, ok := newGlobal[n]; !ok || nf.NewerThan(lf) {
newGlobal[n] = nf
available[n] = []string{nodeID}
if nf.Modified > highestMod {
highestMod = nf.Modified
}
} else if lf.Equals(nf) {
available[n] = append(available[n], nodeID)
}
}
}
m.rmut.RUnlock()
for f, ns := range available {
m.fq.SetAvailable(f, ns)
}
// Figure out if anything actually changed
m.gmut.RLock()
var updated bool
if highestMod > m.updateGlobal || len(newGlobal) != len(m.global) {
updated = true
} else {
for n, f0 := range newGlobal {
if f1, ok := m.global[n]; !ok || !f0.Equals(f1) {
updated = true
break
}
}
}
m.gmut.RUnlock()
if updated {
m.gmut.Lock()
m.umut.Lock()
m.global = newGlobal
m.updateGlobal = time.Now().Unix()
m.umut.Unlock()
m.gmut.Unlock()
}
}
type addOrder struct {
n string
remote []scanner.Block
fm *fileMonitor
}
func (m *Model) recomputeNeedForGlobal() {
var toDelete []scanner.File
var toAdd []addOrder
m.gmut.RLock()
for _, gf := range m.global {
toAdd, toDelete = m.recomputeNeedForFile(gf, toAdd, toDelete)
}
m.gmut.RUnlock()
for _, ao := range toAdd {
m.fq.Add(ao.n, ao.remote, ao.fm)
}
for _, gf := range toDelete {
m.dq <- gf
}
}
func (m *Model) recomputeNeedForFiles(files []scanner.File) {
var toDelete []scanner.File
var toAdd []addOrder
m.gmut.RLock()
for _, gf := range files {
toAdd, toDelete = m.recomputeNeedForFile(gf, toAdd, toDelete)
}
m.gmut.RUnlock()
for _, ao := range toAdd {
m.fq.Add(ao.n, ao.remote, ao.fm)
}
for _, gf := range toDelete {
m.dq <- gf
}
}
func (m *Model) recomputeNeedForFile(gf scanner.File, toAdd []addOrder, toDelete []scanner.File) ([]addOrder, []scanner.File) {
m.lmut.RLock()
lf, ok := m.local[gf.Name]
m.lmut.RUnlock()
if !ok || gf.NewerThan(lf) {
if gf.Suppressed {
// Never attempt to sync invalid files
return toAdd, toDelete
}
if gf.Flags&protocol.FlagDeleted != 0 && !m.delete {
// Don't want to delete files, so forget this need
return toAdd, toDelete
}
if gf.Flags&protocol.FlagDeleted != 0 && !ok {
// Don't have the file, so don't need to delete it
return toAdd, toDelete
}
if debugNeed {
dlog.Printf("need: lf:%v gf:%v", lf, gf)
}
if gf.Flags&protocol.FlagDeleted != 0 {
toDelete = append(toDelete, gf)
} else {
local, remote := scanner.BlockDiff(lf.Blocks, gf.Blocks)
fm := fileMonitor{
name: FSNormalize(gf.Name),
path: FSNormalize(path.Clean(path.Join(m.dir, gf.Name))),
global: gf,
model: m,
localBlocks: local,
}
toAdd = append(toAdd, addOrder{gf.Name, remote, &fm})
}
}
return toAdd, toDelete
}
func (m *Model) WhoHas(name string) []string {
var remote []string
m.gmut.RLock()
m.rmut.RLock()
gf := m.global[name]
for node, files := range m.remote {
if file, ok := files[name]; ok && file.Equals(gf) {
remote = append(remote, node)
}
}
m.rmut.RUnlock()
m.gmut.RUnlock()
return remote
}
func (m *Model) deleteLoop() {
for file := range m.dq {
if debugPull {
dlog.Println("delete", file.Name)
}
path := FSNormalize(path.Clean(path.Join(m.dir, file.Name)))
err := os.Remove(path)
if err != nil {
warnf("%s: %v", file.Name, err)
}
m.updateLocal(file)
indexWg.Wait()
}
}
@@ -903,7 +460,8 @@ func fileFromFileInfo(f protocol.FileInfo) scanner.File {
offset += int64(b.Size)
}
return scanner.File{
Name: f.Name,
// Name is with native separator and normalization
Name: filepath.FromSlash(f.Name),
Size: offset,
Flags: f.Flags &^ protocol.FlagInvalid,
Modified: f.Modified,
@@ -922,7 +480,7 @@ func fileInfoFromFile(f scanner.File) protocol.FileInfo {
}
}
pf := protocol.FileInfo{
Name: f.Name,
Name: filepath.ToSlash(f.Name),
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,

363
cmd/syncthing/model_test.go
View File

@@ -4,30 +4,14 @@ import (
"bytes"
"fmt"
"os"
"reflect"
"testing"
"time"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
func TestNewModel(t *testing.T) {
m := NewModel("foo", 1e6)
if m == nil {
t.Fatalf("NewModel returned nil")
}
if fs, _ := m.NeedFiles(); len(fs) > 0 {
t.Errorf("New model should have no Need")
}
if len(m.local) > 0 {
t.Errorf("New model should have no Have")
}
}
var testDataExpected = map[string]scanner.File{
"foo": scanner.File{
Name: "foo",
@@ -62,295 +46,6 @@ func init() {
}
}
func TestUpdateLocal(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if fs, _ := m.NeedFiles(); len(fs) > 0 {
t.Fatalf("Model with only local data should have no need")
}
if l1, l2 := len(m.local), len(testDataExpected); l1 != l2 {
t.Fatalf("Model len(local) incorrect, %d != %d", l1, l2)
}
if l1, l2 := len(m.global), len(testDataExpected); l1 != l2 {
t.Fatalf("Model len(global) incorrect, %d != %d", l1, l2)
}
for name, file := range testDataExpected {
if f, ok := m.local[name]; ok {
if !reflect.DeepEqual(f, file) {
t.Errorf("Incorrect local\n%v !=\n%v\nfor file %q", f, file, name)
}
} else {
t.Errorf("Missing file %q in local table", name)
}
if f, ok := m.global[name]; ok {
if !reflect.DeepEqual(f, file) {
t.Errorf("Incorrect global\n%v !=\n%v\nfor file %q", f, file, name)
}
} else {
t.Errorf("Missing file %q in global table", name)
}
}
for _, f := range fs {
if hf, ok := m.local[f.Name]; !ok || hf.Modified != f.Modified {
t.Fatalf("Incorrect local for %q", f.Name)
}
if cf, ok := m.global[f.Name]; !ok || cf.Modified != f.Modified {
t.Fatalf("Incorrect global for %q", f.Name)
}
}
}
func TestRemoteUpdateExisting(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
Name: "foo",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("42", []protocol.FileInfo{newFile})
if fs, _ := m.NeedFiles(); len(fs) != 1 {
t.Errorf("Model missing Need for one file (%d != 1)", len(fs))
}
}
func TestRemoteAddNew(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
Name: "a new file",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("42", []protocol.FileInfo{newFile})
if fs, _ := m.NeedFiles(); len(fs) != 1 {
t.Errorf("Model len(m.need) incorrect (%d != 1)", len(fs))
}
}
func TestRemoteUpdateOld(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
oldTimeStamp := int64(1234)
newFile := protocol.FileInfo{
Name: "foo",
Modified: oldTimeStamp,
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("42", []protocol.FileInfo{newFile})
if fs, _ := m.NeedFiles(); len(fs) != 0 {
t.Errorf("Model len(need) incorrect (%d != 0)", len(fs))
}
}
func TestRemoteIndexUpdate(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
foo := protocol.FileInfo{
Name: "foo",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
bar := protocol.FileInfo{
Name: "bar",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("42", []protocol.FileInfo{foo})
if fs, _ := m.NeedFiles(); fs[0].Name != "foo" {
t.Error("Model doesn't need 'foo'")
}
m.IndexUpdate("42", []protocol.FileInfo{bar})
if fs, _ := m.NeedFiles(); fs[0].Name != "foo" {
t.Error("Model doesn't need 'foo'")
}
if fs, _ := m.NeedFiles(); fs[1].Name != "bar" {
t.Error("Model doesn't need 'bar'")
}
}
func TestDelete(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs); l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
ot := time.Now().Unix()
newFile := scanner.File{
Name: "a new file",
Modified: ot,
Blocks: []scanner.Block{{0, 100, []byte("some hash bytes")}},
}
m.updateLocal(newFile)
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
// The deleted file is kept in the local and global tables and marked as deleted.
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
if m.local["a new file"].Flags&(1<<12) == 0 {
t.Error("Unexpected deleted flag = 0 in local table")
}
if len(m.local["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in local")
}
if ft := m.local["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
}
if m.global["a new file"].Flags&(1<<12) == 0 {
t.Error("Unexpected deleted flag = 0 in global table")
}
if len(m.global["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in global")
}
if ft := m.global["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot+1)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
}
// Another update should change nothing
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
if m.local["a new file"].Flags&(1<<12) == 0 {
t.Error("Unexpected deleted flag = 0 in local table")
}
if len(m.local["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in local")
}
if ft := m.local["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
}
if m.global["a new file"].Flags&(1<<12) == 0 {
t.Error("Unexpected deleted flag = 0 in global table")
}
if len(m.global["a new file"].Blocks) != 0 {
t.Error("Unexpected non-zero blocks for deleted file in global")
}
if ft := m.global["a new file"].Modified; ft != ot {
t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot)
}
if fv := m.local["a new file"].Version; fv != 1 {
t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
}
}
func TestForgetNode(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs); l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
if fs, _ := m.NeedFiles(); len(fs) != 0 {
t.Errorf("Model len(need) incorrect (%d != 0)", len(fs))
}
newFile := protocol.FileInfo{
Name: "new file",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("42", []protocol.FileInfo{newFile})
newFile = protocol.FileInfo{
Name: "new file 2",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
}
m.Index("43", []protocol.FileInfo{newFile})
if l1, l2 := len(m.local), len(fs); l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs)+2; l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
if fs, _ := m.NeedFiles(); len(fs) != 2 {
t.Errorf("Model len(need) incorrect (%d != 2)", len(fs))
}
m.Close("42", nil)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
}
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
}
if fs, _ := m.NeedFiles(); len(fs) != 1 {
t.Errorf("Model len(need) incorrect (%d != 1)", len(fs))
}
}
func TestRequest(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
@@ -374,36 +69,6 @@ func TestRequest(t *testing.T) {
}
}
func TestIgnoreWithUnknownFlags(t *testing.T) {
m := NewModel("testdata", 1e6)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
valid := protocol.FileInfo{
Name: "valid",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
Flags: protocol.FlagDeleted | 0755,
}
invalid := protocol.FileInfo{
Name: "invalid",
Modified: time.Now().Unix(),
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
Flags: 1<<27 | protocol.FlagDeleted | 0755,
}
m.Index("42", []protocol.FileInfo{valid, invalid})
if _, ok := m.global[valid.Name]; !ok {
t.Error("Model should include", valid)
}
if _, ok := m.global[invalid.Name]; ok {
t.Error("Model not should include", invalid)
}
}
func genFiles(n int) []protocol.FileInfo {
files := make([]protocol.FileInfo, n)
t := time.Now().Unix()
@@ -554,3 +219,29 @@ func BenchmarkRequest(b *testing.B) {
}
}
}
func TestActivityMap(t *testing.T) {
cm := cid.NewMap()
fooID := cm.Get("foo")
if fooID == 0 {
t.Fatal("ID cannot be zero")
}
barID := cm.Get("bar")
if barID == 0 {
t.Fatal("ID cannot be zero")
}
m := make(activityMap)
if node := m.leastBusyNode(1<<fooID, cm); node != "foo" {
t.Errorf("Incorrect least busy node %q", node)
}
if node := m.leastBusyNode(1<<barID, cm); node != "bar" {
t.Errorf("Incorrect least busy node %q", node)
}
if node := m.leastBusyNode(1<<fooID|1<<barID, cm); node != "foo" {
t.Errorf("Incorrect least busy node %q", node)
}
if node := m.leastBusyNode(1<<fooID|1<<barID, cm); node != "bar" {
t.Errorf("Incorrect least busy node %q", node)
}
}

11
cmd/syncthing/normalize.go
View File

@@ -1,11 +0,0 @@
//+build !darwin
package main
import "code.google.com/p/go.text/unicode/norm"
// FSNormalize returns the string with the required unicode normalization for
// the host operating system.
func FSNormalize(s string) string {
return norm.NFC.String(s)
}

11
cmd/syncthing/normalize_darwin.go
View File

@@ -1,11 +0,0 @@
//+build darwin
package main
import "code.google.com/p/go.text/unicode/norm"
// FSNormalize returns the string with the required unicode normalization for
// the host operating system.
func FSNormalize(s string) string {
return norm.NFD.String(s)
}

477
cmd/syncthing/puller.go Normal file
View File

@@ -0,0 +1,477 @@
package main
import (
"bytes"
"errors"
"os"
"path/filepath"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
type requestResult struct {
node string
file scanner.File
filepath string // full filepath name
offset int64
data []byte
err error
}
type openFile struct {
filepath string // full filepath name
temp string // temporary filename
availability uint64 // availability bitset
file *os.File
err error // error when opening or writing to file, all following operations are cancelled
outstanding int // number of requests we still have outstanding
done bool // we have sent all requests for this file
}
type activityMap map[string]int
func (m activityMap) leastBusyNode(availability uint64, cm *cid.Map) string {
var low int = 2<<30 - 1
var selected string
for _, node := range cm.Names() {
id := cm.Get(node)
if id == cid.LocalID {
continue
}
usage := m[node]
if availability&(1<<id) != 0 {
if usage < low {
low = usage
selected = node
}
}
}
m[selected]++
return selected
}
func (m activityMap) decrease(node string) {
m[node]--
}
var errNoNode = errors.New("no available source node")
type puller struct {
repo string
dir string
bq *blockQueue
model *Model
oustandingPerNode activityMap
openFiles map[string]openFile
requestSlots chan bool
blocks chan bqBlock
requestResults chan requestResult
}
func newPuller(repo, dir string, model *Model, slots int) *puller {
p := &puller{
repo: repo,
dir: dir,
bq: newBlockQueue(),
model: model,
oustandingPerNode: make(activityMap),
openFiles: make(map[string]openFile),
requestSlots: make(chan bool, slots),
blocks: make(chan bqBlock),
requestResults: make(chan requestResult),
}
if slots > 0 {
// Read/write
for i := 0; i < slots; i++ {
p.requestSlots <- true
}
if debugPull {
dlog.Printf("starting puller; repo %q dir %q slots %d", repo, dir, slots)
}
go p.run()
} else {
// Read only
if debugPull {
dlog.Printf("starting puller; repo %q dir %q (read only)", repo, dir)
}
go p.runRO()
}
return p
}
func (p *puller) run() {
go func() {
// fill blocks queue when there are free slots
for {
<-p.requestSlots
b := p.bq.get()
if debugPull {
dlog.Printf("filler: queueing %q offset %d copy %d", b.file.Name, b.block.Offset, len(b.copy))
}
p.blocks <- b
}
}()
walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
timeout := time.Tick(5 * time.Second)
sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
w := &scanner.Walker{
Dir: p.dir,
IgnoreFile: ".stignore",
FollowSymlinks: cfg.Options.FollowSymlinks,
BlockSize: BlockSize,
TempNamer: defTempNamer,
Suppressor: sup,
CurrentFiler: p.model,
}
for {
// Run the pulling loop as long as there are blocks to fetch
pull:
for {
select {
case res := <-p.requestResults:
p.requestSlots <- true
p.handleRequestResult(res)
case b := <-p.blocks:
p.handleBlock(b)
case <-timeout:
if debugPull {
dlog.Println("timeout")
}
if len(p.openFiles) == 0 && p.bq.empty() {
// Nothing more to do for the moment
break pull
}
if debugPull {
dlog.Printf("idle but have %d open files", len(p.openFiles))
i := 5
for _, f := range p.openFiles {
dlog.Printf(" %v", f)
i--
if i == 0 {
break
}
}
}
}
}
// Do a rescan if it's time for it
select {
case <-walkTicker:
if debugPull {
dlog.Println("time for rescan")
}
files, _ := w.Walk()
p.model.fs.ReplaceWithDelete(cid.LocalID, files)
default:
}
// Queue more blocks to fetch, if any
p.queueNeededBlocks()
}
}
func (p *puller) runRO() {
walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
w := &scanner.Walker{
Dir: p.dir,
IgnoreFile: ".stignore",
FollowSymlinks: cfg.Options.FollowSymlinks,
BlockSize: BlockSize,
TempNamer: defTempNamer,
Suppressor: sup,
CurrentFiler: p.model,
}
for _ = range walkTicker {
if debugPull {
dlog.Println("time for rescan")
}
files, _ := w.Walk()
p.model.fs.ReplaceWithDelete(cid.LocalID, files)
}
}
func (p *puller) handleRequestResult(res requestResult) {
p.oustandingPerNode.decrease(res.node)
f := res.file
of, ok := p.openFiles[f.Name]
if !ok || of.err != nil {
// no entry in openFiles means there was an error and we've cancelled the operation
return
}
_, of.err = of.file.WriteAt(res.data, res.offset)
buffers.Put(res.data)
of.outstanding--
p.openFiles[f.Name] = of
if debugPull {
dlog.Printf("pull: wrote %q offset %d outstanding %d done %v", f.Name, res.offset, of.outstanding, of.done)
}
if of.done && of.outstanding == 0 {
if debugPull {
dlog.Printf("pull: closing %q", f.Name)
}
of.file.Close()
defer os.Remove(of.temp)
delete(p.openFiles, f.Name)
fd, err := os.Open(of.temp)
if err != nil {
if debugPull {
dlog.Printf("pull: error: %q: %v", f.Name, err)
}
return
}
hb, _ := scanner.Blocks(fd, BlockSize)
fd.Close()
if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
if debugPull {
dlog.Printf("pull: %q: nblocks %d != %d", f.Name, l0, l1)
}
return
}
for i := range hb {
if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
dlog.Printf("pull: %q: block %d hash mismatch", f.Name, i)
return
}
}
t := time.Unix(f.Modified, 0)
os.Chtimes(of.temp, t, t)
os.Chmod(of.temp, os.FileMode(f.Flags&0777))
if debugPull {
dlog.Printf("pull: rename %q: %q", f.Name, of.filepath)
}
if err := Rename(of.temp, of.filepath); err == nil {
p.model.fs.Update(cid.LocalID, []scanner.File{f})
} else {
dlog.Printf("pull: error: %q: %v", f.Name, err)
}
}
}
func (p *puller) handleBlock(b bqBlock) {
f := b.file
of, ok := p.openFiles[f.Name]
of.done = b.last
if !ok {
if debugPull {
dlog.Printf("pull: opening file %q", f.Name)
}
of.availability = uint64(p.model.fs.Availability(f.Name))
of.filepath = filepath.Join(p.dir, f.Name)
of.temp = filepath.Join(p.dir, defTempNamer.TempName(f.Name))
dirName := filepath.Dir(of.filepath)
_, err := os.Stat(dirName)
if err != nil {
err = os.MkdirAll(dirName, 0777)
}
if err != nil {
dlog.Printf("pull: error: %q: %v", f.Name, err)
}
of.file, of.err = os.Create(of.temp)
if of.err != nil {
if debugPull {
dlog.Printf("pull: error: %q: %v", f.Name, of.err)
}
if !b.last {
p.openFiles[f.Name] = of
}
p.requestSlots <- true
return
}
}
if of.err != nil {
// We have already failed this file.
if debugPull {
dlog.Printf("pull: error: %q has already failed: %v", f.Name, of.err)
}
if b.last {
dlog.Printf("pull: removing failed file %q", f.Name)
delete(p.openFiles, f.Name)
}
p.requestSlots <- true
return
}
p.openFiles[f.Name] = of
switch {
case len(b.copy) > 0:
p.handleCopyBlock(b)
p.requestSlots <- true
case b.block.Size > 0:
p.handleRequestBlock(b)
// Request slot gets freed in <-p.blocks case
default:
p.handleEmptyBlock(b)
p.requestSlots <- true
}
}
func (p *puller) handleCopyBlock(b bqBlock) {
// We have blocks to copy from the existing file
f := b.file
of := p.openFiles[f.Name]
if debugPull {
dlog.Printf("pull: copying %d blocks for %q", len(b.copy), f.Name)
}
var exfd *os.File
exfd, of.err = os.Open(of.filepath)
if of.err != nil {
if debugPull {
dlog.Printf("pull: error: %q: %v", f.Name, of.err)
}
of.file.Close()
of.file = nil
p.openFiles[f.Name] = of
return
}
defer exfd.Close()
for _, b := range b.copy {
bs := buffers.Get(int(b.Size))
_, of.err = exfd.ReadAt(bs, b.Offset)
if of.err == nil {
_, of.err = of.file.WriteAt(bs, b.Offset)
}
buffers.Put(bs)
if of.err != nil {
if debugPull {
dlog.Printf("pull: error: %q: %v", f.Name, of.err)
}
exfd.Close()
of.file.Close()
of.file = nil
p.openFiles[f.Name] = of
return
}
}
}
func (p *puller) handleRequestBlock(b bqBlock) {
// We have a block to get from the network
f := b.file
of := p.openFiles[f.Name]
node := p.oustandingPerNode.leastBusyNode(of.availability, p.model.cm)
if len(node) == 0 {
of.err = errNoNode
if of.file != nil {
of.file.Close()
of.file = nil
os.Remove(of.temp)
}
if b.last {
delete(p.openFiles, f.Name)
} else {
p.openFiles[f.Name] = of
}
p.requestSlots <- true
return
}
of.outstanding++
p.openFiles[f.Name] = of
go func(node string, b bqBlock) {
if debugPull {
dlog.Printf("pull: requesting %q offset %d size %d from %q outstanding %d", f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
}
bs, err := p.model.requestGlobal(node, f.Name, b.block.Offset, int(b.block.Size), nil)
p.requestResults <- requestResult{
node: node,
file: f,
filepath: of.filepath,
offset: b.block.Offset,
data: bs,
err: err,
}
}(node, b)
}
func (p *puller) handleEmptyBlock(b bqBlock) {
f := b.file
of := p.openFiles[f.Name]
if b.last {
if of.err == nil {
of.file.Close()
}
}
if f.Flags&protocol.FlagDeleted != 0 {
if debugPull {
dlog.Printf("pull: delete %q", f.Name)
}
os.Remove(of.temp)
os.Remove(of.filepath)
} else {
if debugPull {
dlog.Printf("pull: no blocks to fetch and nothing to copy for %q", f.Name)
}
t := time.Unix(f.Modified, 0)
os.Chtimes(of.temp, t, t)
os.Chmod(of.temp, os.FileMode(f.Flags&0777))
Rename(of.temp, of.filepath)
}
delete(p.openFiles, f.Name)
p.model.fs.Update(cid.LocalID, []scanner.File{f})
}
func (p *puller) queueNeededBlocks() {
queued := 0
for _, f := range p.model.fs.Need(cid.LocalID) {
lf := p.model.fs.Get(cid.LocalID, f.Name)
have, need := scanner.BlockDiff(lf.Blocks, f.Blocks)
if debugNeed {
dlog.Printf("need:\n local: %v\n global: %v\n haveBlocks: %v\n needBlocks: %v", lf, f, have, need)
}
queued++
p.bq.put(bqAdd{
file: f,
have: have,
need: need,
})
}
if debugPull && queued > 0 {
dlog.Printf("queued %d blocks", queued)
}
}

BIN
cmd/syncthing/syncthing Executable file
View File

Binary file not shown.

13
cmd/syncthing/tempname.go
View File

@@ -2,9 +2,7 @@ package main
import (
"fmt"
"path"
"path/filepath"
"runtime"
"strings"
)
@@ -15,14 +13,11 @@ type tempNamer struct {
var defTempNamer = tempNamer{".syncthing"}
func (t tempNamer) IsTemporary(name string) bool {
if runtime.GOOS == "windows" {
name = filepath.ToSlash(name)
}
return strings.HasPrefix(path.Base(name), t.prefix)
return strings.HasPrefix(filepath.Base(name), t.prefix)
}
func (t tempNamer) TempName(name string) string {
tdir := path.Dir(name)
tname := fmt.Sprintf("%s.%s", t.prefix, path.Base(name))
return path.Join(tdir, tname)
tdir := filepath.Dir(name)
tname := fmt.Sprintf("%s.%s", t.prefix, filepath.Base(name))
return filepath.Join(tdir, tname)
}

8
cmd/syncthing/tls.go
View File

@@ -11,7 +11,7 @@ import (
"encoding/pem"
"math/big"
"os"
"path"
"path/filepath"
"strings"
"time"
)
@@ -22,7 +22,7 @@ const (
)
func loadCert(dir string) (tls.Certificate, error) {
return tls.LoadX509KeyPair(path.Join(dir, "cert.pem"), path.Join(dir, "key.pem"))
return tls.LoadX509KeyPair(filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem"))
}
func certID(bs []byte) string {
@@ -57,13 +57,13 @@ func newCertificate(dir string) {
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
fatalErr(err)
certOut, err := os.Create(path.Join(dir, "cert.pem"))
certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
fatalErr(err)
pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
certOut.Close()
okln("Created RSA certificate file")
keyOut, err := os.OpenFile(path.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
fatalErr(err)
pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
keyOut.Close()

16
cmd/syncthing/util.go
View File

@@ -1,6 +1,10 @@
package main
import "fmt"
import (
"fmt"
"os"
"runtime"
)
func MetricPrefix(n int64) string {
if n > 1e9 {
@@ -27,3 +31,13 @@ func BinaryPrefix(n int64) string {
}
return fmt.Sprintf("%d ", n)
}
func Rename(from, to string) error {
if runtime.GOOS == "windows" {
err := os.Remove(to)
if err != nil && !os.IsNotExist(err) {
warnln(err)
}
}
return os.Rename(from, to)
}