Complete rewrite of the puller (fixes #638, fixes #715, fixes #701)

check-contrib.sh

@@ -8,6 +8,8 @@ missing-contribs() {
 
 no-docs-typos() {
 	# Commits that are known to not change code
+	grep -v 63bd0136fb40a91efaa279cb4b4159d82e8e6904 |\
+	grep -v 4e2feb6fbc791bb8a2daf0ab8efb10775d66343e |\
 	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
 	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
 	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\

cmd/syncthing/gui.go

@@ -87,7 +87,6 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	getRestMux.HandleFunc("/rest/ignores", withModel(m, restGetIgnores))
 	getRestMux.HandleFunc("/rest/lang", restGetLang)
 	getRestMux.HandleFunc("/rest/model", withModel(m, restGetModel))
-	getRestMux.HandleFunc("/rest/model/version", withModel(m, restGetModelVersion))
 	getRestMux.HandleFunc("/rest/need", withModel(m, restGetNeed))
 	getRestMux.HandleFunc("/rest/nodeid", restGetNodeID)
 	getRestMux.HandleFunc("/rest/report", withModel(m, restGetReport))
@@ -238,17 +237,6 @@ func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetModelVersion(m *model.Model, w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
-	var res = make(map[string]interface{})
-
-	res["version"] = m.LocalVersion(repo)
-
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(res)
-}
-
 func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	var repo = qs.Get("repo")
@@ -273,7 +261,7 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	res["inSyncFiles"], res["inSyncBytes"] = globalFiles-needFiles, globalBytes-needBytes
 
 	res["state"], res["stateChanged"] = m.State(repo)
-	res["version"] = m.LocalVersion(repo)
+	res["version"] = m.CurrentLocalVersion(repo) + m.RemoteLocalVersion(repo)
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)

cmd/syncthing/main.go

@@ -442,7 +442,7 @@ nextRepo:
 		m.AddRepo(repo)
 
 		fi, err := os.Stat(repo.Directory)
-		if m.LocalVersion(repo.ID) > 0 {
+		if m.CurrentLocalVersion(repo.ID) > 0 {
 			// Safety check. If the cached index contains files but the
 			// repository doesn't exist, we have a problem. We would assume
 			// that all files have been deleted which might not be the case,
@@ -453,8 +453,8 @@ nextRepo:
 				continue nextRepo
 			}
 		} else if os.IsNotExist(err) {
-			// If we don't have ny files in the index, and the directory does
+			// If we don't have any files in the index, and the directory
-			// exist, try creating it.
+			// doesn't exist, try creating it.
 			err = os.MkdirAll(repo.Directory, 0700)
 		}
 
@@ -582,7 +582,7 @@ nextRepo:
 			m.StartRepoRO(repo.ID)
 		} else {
 			l.Okf("Ready to synchronize %s (read-write)", repo.ID)
-			m.StartRepoRW(repo.ID, cfg.Options.ParallelRequests)
+			m.StartRepoRW(repo.ID)
 		}
 	}
 
@@ -1159,7 +1159,7 @@ func standbyMonitor() {
 	for {
 		time.Sleep(10 * time.Second)
 		if time.Since(now) > 2*time.Minute {
-			l.Infoln("Paused state detected, possibly woke up from standby. Restarting in", restartDelay)
+			l.Infof("Paused state detected, possibly woke up from standby. Restarting in %v.", restartDelay)
 
 			// We most likely just woke from standby. If we restart
 			// immediately chances are we won't have networking ready. Give

internal/config/config.go

@@ -118,7 +118,6 @@ type OptionsConfiguration struct {
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
 	LocalAnnPort       int      `xml:"localAnnouncePort" default:"21025"`
 	LocalAnnMCAddr     string   `xml:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
-	ParallelRequests   int      `xml:"parallelRequests" default:"16"`
 	MaxSendKbps        int      `xml:"maxSendKbps"`
 	MaxRecvKbps        int      `xml:"maxRecvKbps"`
 	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`

internal/config/config_test.go

@@ -29,7 +29,6 @@ func TestDefaultValues(t *testing.T) {
 		LocalAnnEnabled:    true,
 		LocalAnnPort:       21025,
 		LocalAnnMCAddr:     "[ff32::5222]:21026",
-		ParallelRequests:   16,
 		MaxSendKbps:        0,
 		MaxRecvKbps:        0,
 		ReconnectIntervalS: 60,
@@ -121,7 +120,6 @@ func TestOverriddenValues(t *testing.T) {
 		LocalAnnEnabled:    false,
 		LocalAnnPort:       42123,
 		LocalAnnMCAddr:     "quux:3232",
-		ParallelRequests:   32,
 		MaxSendKbps:        1234,
 		MaxRecvKbps:        2341,
 		ReconnectIntervalS: 6000,

internal/model/model.go

@@ -28,6 +28,7 @@ import (
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syncthing/syncthing/internal/scanner"
 	"github.com/syncthing/syncthing/internal/stats"
+	"github.com/syncthing/syncthing/internal/versioner"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -138,22 +139,54 @@ func NewModel(indexDir string, cfg *config.Configuration, nodeName, clientName,
 // StartRW starts read/write processing on the current model. When in
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
-func (m *Model) StartRepoRW(repo string, threads int) {
+func (m *Model) StartRepoRW(repo string) {
-	m.rmut.RLock()
+	m.rmut.Lock()
-	defer m.rmut.RUnlock()
+	cfg, ok := m.repoCfgs[repo]
+	m.rmut.Unlock()
 
-	if cfg, ok := m.repoCfgs[repo]; !ok {
+	if !ok {
-		panic("cannot start without repo")
+		panic("cannot start nonexistent repo " + repo)
-	} else {
-		newPuller(cfg, m, threads, m.cfg)
 	}
+
+	p := Puller{
+		repo:     repo,
+		dir:      cfg.Directory,
+		scanIntv: time.Duration(cfg.RescanIntervalS) * time.Second,
+		model:    m,
+	}
+
+	if len(cfg.Versioning.Type) > 0 {
+		factory, ok := versioner.Factories[cfg.Versioning.Type]
+		if !ok {
+			l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
+		}
+		p.versioner = factory(repo, cfg.Directory, cfg.Versioning.Params)
+	}
+
+	go p.Serve()
 }
 
 // StartRO starts read only processing on the current model. When in
 // read only mode the model will announce files to the cluster but not
 // pull in any external changes.
 func (m *Model) StartRepoRO(repo string) {
-	m.StartRepoRW(repo, 0) // zero threads => read only
+	intv := time.Duration(m.repoCfgs[repo].RescanIntervalS) * time.Second
+	go func() {
+		for {
+			time.Sleep(intv)
+
+			if debug {
+				l.Debugln(m, "rescan", repo)
+			}
+
+			m.setState(repo, RepoScanning)
+			if err := m.ScanRepo(repo); err != nil {
+				invalidateRepo(m.cfg, repo, err)
+				return
+			}
+			m.setState(repo, RepoIdle)
+		}
+	}()
 }
 
 type ConnectionInfo struct {
@@ -240,7 +273,7 @@ func (m *Model) Completion(node protocol.NodeID, repo string) float64 {
 
 	res := 100 * (1 - float64(need)/float64(tot))
 	if debug {
-		l.Debugf("Completion(%s, %q): %f (%d / %d)", node, repo, res, need, tot)
+		l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, node, repo, res, need, tot)
 	}
 
 	return res
@@ -316,7 +349,7 @@ func (m *Model) NeedSize(repo string) (files int, bytes int64) {
 		})
 	}
 	if debug {
-		l.Debugf("NeedSize(%q): %d %d", repo, files, bytes)
+		l.Debugf("%v NeedSize(%q): %d %d", m, repo, files, bytes)
 	}
 	return
 }
@@ -389,7 +422,7 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf
 // Implements the protocol.Model interface.
 func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.FileInfo) {
 	if debug {
-		l.Debugf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
+		l.Debugf("%v IDXUP(in): %s / %q: %d files", m, nodeID, repo, len(fs))
 	}
 
 	if !m.repoSharedWith(repo, nodeID) {
@@ -475,7 +508,7 @@ func (m *Model) ClusterConfig(nodeID protocol.NodeID, cm protocol.ClusterConfigM
 				var id protocol.NodeID
 				copy(id[:], node.ID)
 
-				if m.cfg.GetNodeConfiguration(id)==nil {
+				if m.cfg.GetNodeConfiguration(id) == nil {
 					// The node is currently unknown. Add it to the config.
 
 					l.Infof("Adding node %v to config (vouched for by introducer %v)", id, nodeID)
@@ -574,20 +607,20 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
 	lf := r.Get(protocol.LocalNodeID, name)
 	if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
 		if debug {
-			l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
+			l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, nodeID, repo, name, offset, size, lf)
 		}
 		return nil, ErrInvalid
 	}
 
 	if offset > lf.Size() {
 		if debug {
-			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
+			l.Debugf("%v REQ(in; nonexistent): %s: %q o=%d s=%d", m, nodeID, name, offset, size)
 		}
 		return nil, ErrNoSuchFile
 	}
 
 	if debug && nodeID != protocol.LocalNodeID {
-		l.Debugf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d", m, nodeID, repo, name, offset, size)
 	}
 	m.rmut.RLock()
 	fn := filepath.Join(m.repoCfgs[repo].Directory, name)
@@ -768,15 +801,9 @@ func sendIndexes(conn protocol.Connection, repo string, fs *files.Set, ignores i
 	var err error
 
 	if debug {
-		l.Debugf("sendIndexes for %s-%s@/%q starting", nodeID, name, repo)
+		l.Debugf("sendIndexes for %s-%s/%q starting", nodeID, name, repo)
 	}
 
-	defer func() {
-		if debug {
-			l.Debugf("sendIndexes for %s-%s@/%q exiting: %v", nodeID, name, repo, err)
-		}
-	}()
-
 	minLocalVer, err := sendIndexTo(true, 0, conn, repo, fs, ignores)
 
 	for err == nil {
@@ -787,6 +814,10 @@ func sendIndexes(conn protocol.Connection, repo string, fs *files.Set, ignores i
 
 		minLocalVer, err = sendIndexTo(false, minLocalVer, conn, repo, fs, ignores)
 	}
+
+	if debug {
+		l.Debugf("sendIndexes for %s-%s/%q exiting: %v", nodeID, name, repo, err)
+	}
 }
 
 func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, repo string, fs *files.Set, ignores ignore.Patterns) (uint64, error) {
@@ -877,7 +908,7 @@ func (m *Model) requestGlobal(nodeID protocol.NodeID, repo, name string, offset
 	}
 
 	if debug {
-		l.Debugf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
+		l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x", m, nodeID, repo, name, offset, size, hash)
 	}
 
 	return nc.Request(repo, name, offset, size)
@@ -1175,10 +1206,10 @@ func (m *Model) Override(repo string) {
 	m.setState(repo, RepoIdle)
 }
 
-// Version returns the change version for the given repository. This is
+// CurrentLocalVersion returns the change version for the given repository.
-// guaranteed to increment if the contents of the local or global repository
+// This is guaranteed to increment if the contents of the local repository has
-// has changed.
+// changed.
-func (m *Model) LocalVersion(repo string) uint64 {
+func (m *Model) CurrentLocalVersion(repo string) uint64 {
 	m.rmut.Lock()
 	defer m.rmut.Unlock()
 
@@ -1187,10 +1218,41 @@ func (m *Model) LocalVersion(repo string) uint64 {
 		panic("bug: LocalVersion called for nonexistent repo " + repo)
 	}
 
-	ver := fs.LocalVersion(protocol.LocalNodeID)
+	return fs.LocalVersion(protocol.LocalNodeID)
+}
+
+// RemoteLocalVersion returns the change version for the given repository, as
+// sent by remote peers. This is guaranteed to increment if the contents of
+// the remote or global repository has changed.
+func (m *Model) RemoteLocalVersion(repo string) uint64 {
+	m.rmut.Lock()
+	defer m.rmut.Unlock()
+
+	fs, ok := m.repoFiles[repo]
+	if !ok {
+		panic("bug: LocalVersion called for nonexistent repo " + repo)
+	}
+
+	var ver uint64
 	for _, n := range m.repoNodes[repo] {
 		ver += fs.LocalVersion(n)
 	}
 
 	return ver
 }
+
+func (m *Model) availability(repo string, file string) []protocol.NodeID {
+	m.rmut.Lock()
+	defer m.rmut.Unlock()
+
+	fs, ok := m.repoFiles[repo]
+	if !ok {
+		return nil
+	}
+
+	return fs.Availability(file)
+}
+
+func (m *Model) String() string {
+	return fmt.Sprintf("model@%p", m)
+}

internal/model/model_test.go

@@ -241,25 +241,6 @@ func BenchmarkRequest(b *testing.B) {
 	}
 }
 
-func TestActivityMap(t *testing.T) {
-	isValid := func(protocol.NodeID) bool {
-		return true
-	}
-	m := make(activityMap)
-	if node := m.leastBusyNode([]protocol.NodeID{node1}, isValid); node != node1 {
-		t.Errorf("Incorrect least busy node %q", node)
-	}
-	if node := m.leastBusyNode([]protocol.NodeID{node2}, isValid); node != node2 {
-		t.Errorf("Incorrect least busy node %q", node)
-	}
-	if node := m.leastBusyNode([]protocol.NodeID{node1, node2}, isValid); node != node1 {
-		t.Errorf("Incorrect least busy node %q", node)
-	}
-	if node := m.leastBusyNode([]protocol.NodeID{node1, node2}, isValid); node != node2 {
-		t.Errorf("Incorrect least busy node %q", node)
-	}
-}
-
 func TestNodeRename(t *testing.T) {
 	ccm := protocol.ClusterConfigMessage{
 		ClientName:    "syncthing",

internal/model/nodeactivity.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+// nodeActivity tracks the number of outstanding requests per node and can
+// answer which node is least busy. It is safe for use from multiple
+// goroutines.
+type nodeActivity struct {
+	act map[protocol.NodeID]int
+	mut sync.Mutex
+}
+
+func newNodeActivity() *nodeActivity {
+	return &nodeActivity{
+		act: make(map[protocol.NodeID]int),
+	}
+}
+
+func (m nodeActivity) leastBusy(availability []protocol.NodeID) protocol.NodeID {
+	m.mut.Lock()
+	var low int = 2<<30 - 1
+	var selected protocol.NodeID
+	for _, node := range availability {
+		if usage := m.act[node]; usage < low {
+			low = usage
+			selected = node
+		}
+	}
+	m.mut.Unlock()
+	return selected
+}
+
+func (m nodeActivity) using(node protocol.NodeID) {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+	m.act[node]++
+}
+
+func (m nodeActivity) done(node protocol.NodeID) {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+	m.act[node]--
+}

internal/model/nodeactivity_test.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+func TestNodeActivity(t *testing.T) {
+	n0 := protocol.NodeID{1, 2, 3, 4}
+	n1 := protocol.NodeID{5, 6, 7, 8}
+	n2 := protocol.NodeID{9, 10, 11, 12}
+	nodes := []protocol.NodeID{n0, n1, n2}
+	na := newNodeActivity()
+
+	if lb := na.leastBusy(nodes); lb != n0 {
+		t.Errorf("Least busy node should be n0 (%v) not %v", n0, lb)
+	}
+	if lb := na.leastBusy(nodes); lb != n0 {
+		t.Errorf("Least busy node should still be n0 (%v) not %v", n0, lb)
+	}
+
+	na.using(na.leastBusy(nodes))
+	if lb := na.leastBusy(nodes); lb != n1 {
+		t.Errorf("Least busy node should be n1 (%v) not %v", n1, lb)
+	}
+
+	na.using(na.leastBusy(nodes))
+	if lb := na.leastBusy(nodes); lb != n2 {
+		t.Errorf("Least busy node should be n2 (%v) not %v", n2, lb)
+	}
+
+	na.using(na.leastBusy(nodes))
+	if lb := na.leastBusy(nodes); lb != n0 {
+		t.Errorf("Least busy node should be n0 (%v) not %v", n0, lb)
+	}
+
+	na.done(n1)
+	if lb := na.leastBusy(nodes); lb != n1 {
+		t.Errorf("Least busy node should be n1 (%v) not %v", n1, lb)
+	}
+
+	na.done(n2)
+	if lb := na.leastBusy(nodes); lb != n1 {
+		t.Errorf("Least busy node should still be n1 (%v) not %v", n1, lb)
+	}
+
+	na.done(n0)
+	if lb := na.leastBusy(nodes); lb != n0 {
+		t.Errorf("Least busy node should be n0 (%v) not %v", n0, lb)
+	}
+}

internal/model/sharedpullerstate.go

@@ -0,0 +1,183 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+// A sharedPullerState is kept for each file that is being synced and is kept
+// updated along the way.
+type sharedPullerState struct {
+	// Immutable, does not require locking
+	file     protocol.FileInfo
+	repo     string
+	tempName string
+	realName string
+
+	// Mutable, must be locked for access
+	err        error      // The first error we hit
+	fd         *os.File   // The fd of the temp file
+	copyNeeded int        // Number of copy actions we expect to happen
+	pullNeeded int        // Number of block pulls we expect to happen
+	closed     bool       // Set when the file has been closed
+	mut        sync.Mutex // Protects the above
+}
+
+// tempFile returns the fd for the temporary file, reusing an open fd
+// or creating the file as necessary.
+func (s *sharedPullerState) tempFile() (*os.File, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	// If we've already hit an error, return early
+	if s.err != nil {
+		return nil, s.err
+	}
+
+	// If the temp file is already open, return the file descriptor
+	if s.fd != nil {
+		return s.fd, nil
+	}
+
+	// Ensure that the parent directory exists or can be created
+	dir := filepath.Dir(s.tempName)
+	if info, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(dir, 0755)
+		if err != nil {
+			s.earlyCloseLocked("dst mkdir", err)
+			return nil, err
+		}
+	} else if err != nil {
+		s.earlyCloseLocked("dst stat dir", err)
+		return nil, err
+	} else if !info.IsDir() {
+		err = fmt.Errorf("%q: not a directory", dir)
+		s.earlyCloseLocked("dst mkdir", err)
+		return nil, err
+	} else if info.Mode()&04 == 0 {
+		err := os.Chmod(dir, 0755)
+		if err == nil {
+			defer func() {
+				err := os.Chmod(dir, info.Mode().Perm())
+				if err != nil {
+					panic(err)
+				}
+			}()
+		}
+	}
+
+	// Attempt to create the temp file
+	fd, err := os.OpenFile(s.tempName, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
+	if err != nil {
+		s.earlyCloseLocked("dst create", err)
+		return nil, err
+	}
+
+	// Same fd will be used by all writers
+	s.fd = fd
+
+	return fd, nil
+}
+
+// sourceFile opens the existing source file for reading
+func (s *sharedPullerState) sourceFile() (*os.File, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	// If we've already hit an error, return early
+	if s.err != nil {
+		return nil, s.err
+	}
+
+	// Attempt to open the existing file
+	fd, err := os.Open(s.realName)
+	if err != nil {
+		s.earlyCloseLocked("src open", err)
+		return nil, err
+	}
+
+	return fd, nil
+}
+
+// earlyClose prints a warning message composed of the context and
+// error, and marks the sharedPullerState as failed. Is a no-op when called on
+// an already failed state.
+func (s *sharedPullerState) earlyClose(context string, err error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	s.earlyCloseLocked(context, err)
+}
+
+func (s *sharedPullerState) earlyCloseLocked(context string, err error) {
+	if s.err != nil {
+		return
+	}
+
+	l.Infof("Puller (repo %q, file %q): %s: %v", s.repo, s.file.Name, context, err)
+	s.err = err
+	if s.fd != nil {
+		s.fd.Close()
+		os.Remove(s.tempName)
+	}
+	s.closed = true
+}
+
+func (s *sharedPullerState) failed() error {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	return s.err
+}
+
+func (s *sharedPullerState) copyDone() {
+	s.mut.Lock()
+	s.copyNeeded--
+	if debug {
+		l.Debugln("sharedPullerState", s.repo, s.file.Name, "copyNeeded ->", s.pullNeeded)
+	}
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) pullDone() {
+	s.mut.Lock()
+	s.pullNeeded--
+	if debug {
+		l.Debugln("sharedPullerState", s.repo, s.file.Name, "pullNeeded ->", s.pullNeeded)
+	}
+	s.mut.Unlock()
+}
+
+// finalClose atomically closes and returns closed status of a file. A true
+// first return value means the file was closed and should be finished, with
+// the error indicating the success or failure of the close. A false first
+// return value indicates the file is not ready to be closed, or is already
+// closed and should in either case not be finished off now.
+func (s *sharedPullerState) finalClose() (bool, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	if s.pullNeeded+s.copyNeeded != 0 {
+		// Not done yet.
+		return false, nil
+	}
+	if s.closed {
+		// Already handled.
+		return false, nil
+	}
+
+	s.closed = true
+	if fd := s.fd; fd != nil {
+		s.fd = nil
+		return true, fd.Close()
+	}
+	return true, nil
+}

internal/model/sharedpullerstate_test.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import "testing"
+
+func TestSourceFileOK(t *testing.T) {
+	s := sharedPullerState{
+		realName: "testdata/foo",
+	}
+
+	fd, err := s.sourceFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fd == nil {
+		t.Fatal("Unexpected nil fd")
+	}
+
+	bs := make([]byte, 6)
+	n, err := fd.Read(bs)
+
+	if n != len(bs) {
+		t.Fatal("Wrong read length %d != %d", n, len(bs))
+	}
+	if string(bs) != "foobar" {
+		t.Fatal("Wrong contents %s != foobar", bs)
+	}
+
+	if err := s.failed(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSourceFileBad(t *testing.T) {
+	s := sharedPullerState{
+		realName: "nonexistent",
+	}
+
+	fd, err := s.sourceFile()
+	if err == nil {
+		t.Fatal("Unexpected nil error")
+	}
+	if fd != nil {
+		t.Fatal("Unexpected non-nil fd")
+	}
+	if err := s.failed(); err == nil {
+		t.Fatal("Unexpected nil failed()")
+	}
+}

internal/scanner/blocks.go

@@ -7,6 +7,7 @@ package scanner
 import (
 	"bytes"
 	"crypto/sha256"
+	"fmt"
 	"io"
 
 	"github.com/syncthing/syncthing/internal/protocol"
@@ -88,3 +89,32 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo)
 
 	return have, need
 }
+
+// Verify returns nil or an error describing the mismatch between the block
+// list and actual reader contents
+func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
+	hf := sha256.New()
+	for i, block := range blocks {
+		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
+		_, err := io.Copy(hf, lr)
+		if err != nil {
+			return err
+		}
+
+		hash := hf.Sum(nil)
+		hf.Reset()
+
+		if bytes.Compare(hash, block.Hash) != 0 {
+			return fmt.Errorf("hash mismatch %x != %x for block %d", hash, block.Hash, i)
+		}
+	}
+
+	// We should have reached the end  now
+	bs := make([]byte, 1)
+	n, err := r.Read(bs)
+	if n != 0 || err != io.EOF {
+		return fmt.Errorf("file continues past end of blocks")
+	}
+
+	return nil
+}

internal/scanner/walk_test.go

@@ -133,6 +133,50 @@ func TestWalkError(t *testing.T) {
 	}
 }
 
+func TestVerify(t *testing.T) {
+	blocksize := 16
+	// data should be an even multiple of blocksize long
+	data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut e")
+	buf := bytes.NewBuffer(data)
+
+	blocks, err := Blocks(buf, blocksize, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if exp := len(data) / blocksize; len(blocks) != exp {
+		t.Fatalf("Incorrect number of blocks %d != %d", len(blocks), exp)
+	}
+
+	buf = bytes.NewBuffer(data)
+	err = Verify(buf, blocksize, blocks)
+	t.Log(err)
+	if err != nil {
+		t.Fatal("Unexpected verify failure", err)
+	}
+
+	buf = bytes.NewBuffer(append(data, '\n'))
+	err = Verify(buf, blocksize, blocks)
+	t.Log(err)
+	if err == nil {
+		t.Fatal("Unexpected verify success")
+	}
+
+	buf = bytes.NewBuffer(data[:len(data)-1])
+	err = Verify(buf, blocksize, blocks)
+	t.Log(err)
+	if err == nil {
+		t.Fatal("Unexpected verify success")
+	}
+
+	data[42] = 42
+	buf = bytes.NewBuffer(data)
+	err = Verify(buf, blocksize, blocks)
+	t.Log(err)
+	if err == nil {
+		t.Fatal("Unexpected verify success")
+	}
+}
+
 type fileList []protocol.FileInfo
 
 func (f fileList) Len() int {

test/http.go

@@ -36,7 +36,6 @@ var jsonEndpoints = []string{
 	"/rest/errors",
 	"/rest/events",
 	"/rest/lang",
-	"/rest/model/version?repo=default",
 	"/rest/model?repo=default",
 	"/rest/need",
 	"/rest/nodeid?id=I6KAH7666SLLLB5PFXSOAUFJCDZCYAOMLEKCP2GB32BV5RQST3PSROAU",

test/test-delupd.sh

@@ -19,7 +19,7 @@ go build json.go
 start() {
 	echo "Starting..."
 	for i in 1 2 3 ; do
-		STTRACE=files,model,puller,versioner,protocol STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
+		STTRACE=model,scanner STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
 	done
 }
 
@@ -100,7 +100,7 @@ alterFiles() {
 			echo "  $i: deleting $todelete files..."
 			set +o pipefail
 			find . -type f \
-				| grep -v large \
+				| grep -v timechanged \
 				| sort -k 1.16 \
 				| head -n "$todelete" \
 				| xargs rm -f
@@ -110,11 +110,10 @@ alterFiles() {
 		# Create some new files and alter existing ones
 		echo "  $i: random nonoverlapping"
 		../genfiles -maxexp 22 -files 200
-		echo "  $i: append to large file"
-		dd if=large-$i bs=1024k count=4 >> large-$i 2>/dev/null
 		echo "  $i: new files in ro directory"
 		uuidgen > ro-test/$(uuidgen)
 		chmod 500 ro-test
+		touch "timechanged-$i"
 
 		../md5r -l | sort | grep -v .stversions > ../md5-$i
 		popd >/dev/null
@@ -140,6 +139,7 @@ for i in 1 12-2 23-3; do
 	mkdir ro-test
 	uuidgen > ro-test/$(uuidgen)
 	chmod 500 ro-test
+	dd if=/dev/urandom of="timechanged-$i" bs=1024k count=1
 	popd >/dev/null
 done