mv internal lib

lib/auto/auto_test.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package auto_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/auto"
+)
+
+func TestAssets(t *testing.T) {
+	assets := auto.Assets()
+	idx, ok := assets["index.html"]
+	if !ok {
+		t.Fatal("No index.html in compiled in assets")
+	}
+
+	var gr *gzip.Reader
+	gr, _ = gzip.NewReader(bytes.NewReader(idx))
+	idx, _ = ioutil.ReadAll(gr)
+
+	if !bytes.Contains(idx, []byte("<html")) {
+		t.Fatal("No html in index.html")
+	}
+}

lib/auto/doc.go

@@ -0,0 +1,8 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package auto contains auto generated files for web assets.
+package auto

lib/beacon/beacon.go

@@ -0,0 +1,43 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package beacon
+
+import "net"
+
+type recv struct {
+	data []byte
+	src  net.Addr
+}
+
+type Interface interface {
+	Send(data []byte)
+	Recv() ([]byte, net.Addr)
+}
+
+func genericReader(conn *net.UDPConn, outbox chan<- recv) {
+	bs := make([]byte, 65536)
+	for {
+		n, addr, err := conn.ReadFrom(bs)
+		if err != nil {
+			l.Warnln("multicast read:", err)
+			return
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
+
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case outbox <- recv{c, addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
+			}
+		}
+	}
+}

lib/beacon/broadcast.go

@@ -0,0 +1,229 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package beacon
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/thejerf/suture"
+)
+
+type Broadcast struct {
+	*suture.Supervisor
+	port   int
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewBroadcast(port int) *Broadcast {
+	b := &Broadcast{
+		Supervisor: suture.New("broadcastBeacon", suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				if debug {
+					l.Debugln(line)
+				}
+			},
+		}),
+		port:   port,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	b.Add(&broadcastReader{
+		port:   port,
+		outbox: b.outbox,
+	})
+	b.Add(&broadcastWriter{
+		port:  port,
+		inbox: b.inbox,
+	})
+
+	return b
+}
+
+func (b *Broadcast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Broadcast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+type broadcastWriter struct {
+	port   int
+	inbox  chan []byte
+	conn   *net.UDPConn
+	failed bool // Have we already logged a failure reason?
+}
+
+func (w *broadcastWriter) Serve() {
+	if debug {
+		l.Debugln(w, "starting")
+		defer l.Debugln(w, "stopping")
+	}
+
+	var err error
+	w.conn, err = net.ListenUDP("udp4", nil)
+	if err != nil {
+		if !w.failed {
+			l.Warnln("Local discovery over IPv4 unavailable:", err)
+			w.failed = true
+		}
+		return
+	}
+	defer w.conn.Close()
+
+	w.failed = false
+
+	for bs := range w.inbox {
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			if debug {
+				l.Debugln("Local discovery (broadcast writer):", err)
+			}
+			continue
+		}
+
+		var dsts []net.IP
+		for _, addr := range addrs {
+			if iaddr, ok := addr.(*net.IPNet); ok && len(iaddr.IP) >= 4 && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
+				baddr := bcast(iaddr)
+				dsts = append(dsts, baddr.IP)
+			}
+		}
+
+		if len(dsts) == 0 {
+			// Fall back to the general IPv4 broadcast address
+			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
+		}
+
+		if debug {
+			l.Debugln("addresses:", dsts)
+		}
+
+		for _, ip := range dsts {
+			dst := &net.UDPAddr{IP: ip, Port: w.port}
+
+			w.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
+			_, err := w.conn.WriteTo(bs, dst)
+			if err, ok := err.(net.Error); ok && err.Timeout() {
+				// Write timeouts should not happen. We treat it as a fatal
+				// error on the socket.
+				l.Infoln("Local discovery (broadcast writer):", err)
+				w.failed = true
+				return
+			} else if err, ok := err.(net.Error); ok && err.Temporary() {
+				// A transient error. Lets hope for better luck in the future.
+				if debug {
+					l.Debugln(err)
+				}
+				continue
+			} else if err != nil {
+				// Some other error that we don't expect. Bail and retry.
+				l.Infoln("Local discovery (broadcast writer):", err)
+				w.failed = true
+				return
+			} else if debug {
+				l.Debugf("sent %d bytes to %s", len(bs), dst)
+			}
+		}
+	}
+}
+
+func (w *broadcastWriter) Stop() {
+	w.conn.Close()
+}
+
+func (w *broadcastWriter) String() string {
+	return fmt.Sprintf("broadcastWriter@%p", w)
+}
+
+type broadcastReader struct {
+	port   int
+	outbox chan recv
+	conn   *net.UDPConn
+	failed bool
+}
+
+func (r *broadcastReader) Serve() {
+	if debug {
+		l.Debugln(r, "starting")
+		defer l.Debugln(r, "stopping")
+	}
+
+	var err error
+	r.conn, err = net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
+	if err != nil {
+		if !r.failed {
+			l.Warnln("Local discovery over IPv4 unavailable:", err)
+			r.failed = true
+		}
+		return
+	}
+	defer r.conn.Close()
+
+	bs := make([]byte, 65536)
+	for {
+		n, addr, err := r.conn.ReadFrom(bs)
+		if err != nil {
+			if !r.failed {
+				l.Infoln("Local discovery (broadcast reader):", err)
+				r.failed = true
+			}
+			return
+		}
+
+		r.failed = false
+
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
+
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case r.outbox <- recv{c, addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
+			}
+		}
+	}
+
+}
+
+func (r *broadcastReader) Stop() {
+	r.conn.Close()
+}
+
+func (r *broadcastReader) String() string {
+	return fmt.Sprintf("broadcastReader@%p", r)
+}
+
+func bcast(ip *net.IPNet) *net.IPNet {
+	var bc = &net.IPNet{}
+	bc.IP = make([]byte, len(ip.IP))
+	copy(bc.IP, ip.IP)
+	bc.Mask = ip.Mask
+
+	offset := len(bc.IP) - len(bc.Mask)
+	for i := range bc.IP {
+		if i-offset >= 0 {
+			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
+		}
+	}
+	return bc
+}

lib/beacon/broadcast_test.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package beacon
+
+import (
+	"net"
+	"testing"
+)
+
+var addrToBcast = []struct {
+	addr, bcast string
+}{
+	{"172.16.32.33/25", "172.16.32.127/25"},
+	{"172.16.32.129/25", "172.16.32.255/25"},
+	{"172.16.32.33/24", "172.16.32.255/24"},
+	{"172.16.32.33/22", "172.16.35.255/22"},
+	{"172.16.32.33/0", "255.255.255.255/0"},
+	{"172.16.32.33/32", "172.16.32.33/32"},
+}
+
+func TestBroadcastAddr(t *testing.T) {
+	for _, tc := range addrToBcast {
+		_, net, err := net.ParseCIDR(tc.addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bc := bcast(net).String()
+		if bc != tc.bcast {
+			t.Errorf("%q != %q", bc, tc.bcast)
+		}
+	}
+}

lib/beacon/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package beacon
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

lib/beacon/doc.go

@@ -0,0 +1,8 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package beacon implements an UDP broadcast beacon
+package beacon

lib/beacon/multicast.go

@@ -0,0 +1,66 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package beacon
+
+import "net"
+
+type Multicast struct {
+	conn   *net.UDPConn
+	addr   *net.UDPAddr
+	intf   *net.Interface
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewMulticast(addr, ifname string) (*Multicast, error) {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
+	if err != nil {
+		return nil, err
+	}
+	intf, err := net.InterfaceByName(ifname)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := net.ListenMulticastUDP("udp6", intf, gaddr)
+	if err != nil {
+		return nil, err
+	}
+	b := &Multicast{
+		conn:   conn,
+		addr:   gaddr,
+		intf:   intf,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go genericReader(b.conn, b.outbox)
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Multicast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Multicast) writer() {
+	addr := *b.addr
+	addr.Zone = b.intf.Name
+	for bs := range b.inbox {
+		_, err := b.conn.WriteTo(bs, &addr)
+		if err != nil && debug {
+			l.Debugln(err, "on write to", addr)
+		} else if debug {
+			l.Debugf("sent %d bytes to %s", len(bs), addr.String())
+		}
+	}
+}

lib/config/commit_test.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"errors"
+	"testing"
+)
+
+type requiresRestart struct{}
+
+func (requiresRestart) VerifyConfiguration(_, _ Configuration) error {
+	return nil
+}
+func (requiresRestart) CommitConfiguration(_, _ Configuration) bool {
+	return false
+}
+func (requiresRestart) String() string {
+	return "requiresRestart"
+}
+
+type validationError struct{}
+
+func (validationError) VerifyConfiguration(_, _ Configuration) error {
+	return errors.New("some error")
+}
+func (validationError) CommitConfiguration(_, _ Configuration) bool {
+	return true
+}
+func (validationError) String() string {
+	return "validationError"
+}
+
+func TestReplaceCommit(t *testing.T) {
+	w := Wrap("/dev/null", Configuration{Version: 0})
+	if w.Raw().Version != 0 {
+		t.Fatal("Config incorrect")
+	}
+
+	// Replace config. We should get back a clean response and the config
+	// should change.
+
+	resp := w.Replace(Configuration{Version: 1})
+	if resp.ValidationError != nil {
+		t.Fatal("Should not have a validation error")
+	}
+	if resp.RequiresRestart {
+		t.Fatal("Should not require restart")
+	}
+	if w.Raw().Version != 1 {
+		t.Fatal("Config should have changed")
+	}
+
+	// Now with a subscriber requiring restart. We should get a clean response
+	// but with the restart flag set, and the config should change.
+
+	w.Subscribe(requiresRestart{})
+
+	resp = w.Replace(Configuration{Version: 2})
+	if resp.ValidationError != nil {
+		t.Fatal("Should not have a validation error")
+	}
+	if !resp.RequiresRestart {
+		t.Fatal("Should require restart")
+	}
+	if w.Raw().Version != 2 {
+		t.Fatal("Config should have changed")
+	}
+
+	// Now with a subscriber that throws a validation error. The config should
+	// not change.
+
+	w.Subscribe(validationError{})
+
+	resp = w.Replace(Configuration{Version: 3})
+	if resp.ValidationError == nil {
+		t.Fatal("Should have a validation error")
+	}
+	if resp.RequiresRestart {
+		t.Fatal("Should not require restart")
+	}
+	if w.Raw().Version != 2 {
+		t.Fatal("Config should not have changed")
+	}
+}

lib/config/config.go

@@ -0,0 +1,730 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package config implements reading and writing of the syncthing configuration file.
+package config
+
+import (
+	"encoding/xml"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"golang.org/x/crypto/bcrypt"
+)
+
+const (
+	OldestHandledVersion = 5
+	CurrentVersion       = 10
+	MaxRescanIntervalS   = 365 * 24 * 60 * 60
+)
+
+type Configuration struct {
+	Version        int                   `xml:"version,attr" json:"version"`
+	Folders        []FolderConfiguration `xml:"folder" json:"folders"`
+	Devices        []DeviceConfiguration `xml:"device" json:"devices"`
+	GUI            GUIConfiguration      `xml:"gui" json:"gui"`
+	Options        OptionsConfiguration  `xml:"options" json:"options"`
+	IgnoredDevices []protocol.DeviceID   `xml:"ignoredDevice" json:"ignoredDevices"`
+	XMLName        xml.Name              `xml:"configuration" json:"-"`
+
+	OriginalVersion int `xml:"-" json:"-"` // The version we read from disk, before any conversion
+}
+
+func (cfg Configuration) Copy() Configuration {
+	newCfg := cfg
+
+	// Deep copy FolderConfigurations
+	newCfg.Folders = make([]FolderConfiguration, len(cfg.Folders))
+	for i := range newCfg.Folders {
+		newCfg.Folders[i] = cfg.Folders[i].Copy()
+	}
+
+	// Deep copy DeviceConfigurations
+	newCfg.Devices = make([]DeviceConfiguration, len(cfg.Devices))
+	for i := range newCfg.Devices {
+		newCfg.Devices[i] = cfg.Devices[i].Copy()
+	}
+
+	newCfg.Options = cfg.Options.Copy()
+
+	// DeviceIDs are values
+	newCfg.IgnoredDevices = make([]protocol.DeviceID, len(cfg.IgnoredDevices))
+	copy(newCfg.IgnoredDevices, cfg.IgnoredDevices)
+
+	return newCfg
+}
+
+type FolderConfiguration struct {
+	ID              string                      `xml:"id,attr" json:"id"`
+	RawPath         string                      `xml:"path,attr" json:"path"`
+	Devices         []FolderDeviceConfiguration `xml:"device" json:"devices"`
+	ReadOnly        bool                        `xml:"ro,attr" json:"readOnly"`
+	RescanIntervalS int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
+	IgnorePerms     bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
+	AutoNormalize   bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
+	Versioning      VersioningConfiguration     `xml:"versioning" json:"versioning"`
+	Copiers         int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
+	Pullers         int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
+	Hashers         int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
+	Order           PullOrder                   `xml:"order" json:"order"`
+	IgnoreDelete    bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
+
+	Invalid string `xml:"-" json:"invalid"` // Set at runtime when there is an error, not saved
+}
+
+func (f FolderConfiguration) Copy() FolderConfiguration {
+	c := f
+	c.Devices = make([]FolderDeviceConfiguration, len(f.Devices))
+	copy(c.Devices, f.Devices)
+	return c
+}
+
+func (f FolderConfiguration) Path() string {
+	// This is intentionally not a pointer method, because things like
+	// cfg.Folders["default"].Path() should be valid.
+
+	// Attempt tilde expansion; leave unchanged in case of error
+	if path, err := osutil.ExpandTilde(f.RawPath); err == nil {
+		f.RawPath = path
+	}
+
+	// Attempt absolutification; leave unchanged in case of error
+	if !filepath.IsAbs(f.RawPath) {
+		// Abs() looks like a fairly expensive syscall on Windows, while
+		// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
+		// somewhat faster in the general case, hence the outer if...
+		if path, err := filepath.Abs(f.RawPath); err == nil {
+			f.RawPath = path
+		}
+	}
+
+	// Attempt to enable long filename support on Windows. We may still not
+	// have an absolute path here if the previous steps failed.
+	if runtime.GOOS == "windows" && filepath.IsAbs(f.RawPath) && !strings.HasPrefix(f.RawPath, `\\`) {
+		return `\\?\` + f.RawPath
+	}
+
+	return f.RawPath
+}
+
+func (f *FolderConfiguration) CreateMarker() error {
+	if !f.HasMarker() {
+		marker := filepath.Join(f.Path(), ".stfolder")
+		fd, err := os.Create(marker)
+		if err != nil {
+			return err
+		}
+		fd.Close()
+		osutil.HideFile(marker)
+	}
+
+	return nil
+}
+
+func (f *FolderConfiguration) HasMarker() bool {
+	_, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
+	deviceIDs := make([]protocol.DeviceID, len(f.Devices))
+	for i, n := range f.Devices {
+		deviceIDs[i] = n.DeviceID
+	}
+	return deviceIDs
+}
+
+type VersioningConfiguration struct {
+	Type   string            `xml:"type,attr" json:"type"`
+	Params map[string]string `json:"params"`
+}
+
+type InternalVersioningConfiguration struct {
+	Type   string          `xml:"type,attr,omitempty"`
+	Params []InternalParam `xml:"param"`
+}
+
+type InternalParam struct {
+	Key string `xml:"key,attr"`
+	Val string `xml:"val,attr"`
+}
+
+func (c *VersioningConfiguration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	tmp.Type = c.Type
+	for k, v := range c.Params {
+		tmp.Params = append(tmp.Params, InternalParam{k, v})
+	}
+
+	return e.EncodeElement(tmp, start)
+
+}
+
+func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	err := d.DecodeElement(&tmp, &start)
+	if err != nil {
+		return err
+	}
+
+	c.Type = tmp.Type
+	c.Params = make(map[string]string, len(tmp.Params))
+	for _, p := range tmp.Params {
+		c.Params[p.Key] = p.Val
+	}
+	return nil
+}
+
+type DeviceConfiguration struct {
+	DeviceID    protocol.DeviceID    `xml:"id,attr" json:"deviceID"`
+	Name        string               `xml:"name,attr,omitempty" json:"name"`
+	Addresses   []string             `xml:"address,omitempty" json:"addresses"`
+	Compression protocol.Compression `xml:"compression,attr" json:"compression"`
+	CertName    string               `xml:"certName,attr,omitempty" json:"certName"`
+	Introducer  bool                 `xml:"introducer,attr" json:"introducer"`
+}
+
+func (orig DeviceConfiguration) Copy() DeviceConfiguration {
+	c := orig
+	c.Addresses = make([]string, len(orig.Addresses))
+	copy(c.Addresses, orig.Addresses)
+	return c
+}
+
+type FolderDeviceConfiguration struct {
+	DeviceID protocol.DeviceID `xml:"id,attr" json:"deviceID"`
+}
+
+type OptionsConfiguration struct {
+	ListenAddress           []string `xml:"listenAddress" json:"listenAddress" default:"0.0.0.0:22000"`
+	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"udp4://announce.syncthing.net:22026, udp6://announce-v6.syncthing.net:22026"`
+	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
+	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
+	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21025"`
+	LocalAnnMCAddr          string   `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
+	MaxSendKbps             int      `xml:"maxSendKbps" json:"maxSendKbps"`
+	MaxRecvKbps             int      `xml:"maxRecvKbps" json:"maxRecvKbps"`
+	ReconnectIntervalS      int      `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
+	StartBrowser            bool     `xml:"startBrowser" json:"startBrowser" default:"true"`
+	UPnPEnabled             bool     `xml:"upnpEnabled" json:"upnpEnabled" default:"true"`
+	UPnPLeaseM              int      `xml:"upnpLeaseMinutes" json:"upnpLeaseMinutes" default:"60"`
+	UPnPRenewalM            int      `xml:"upnpRenewalMinutes" json:"upnpRenewalMinutes" default:"30"`
+	UPnPTimeoutS            int      `xml:"upnpTimeoutSeconds" json:"upnpTimeoutSeconds" default:"10"`
+	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
+	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true"`
+	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12"` // 0 for off
+	KeepTemporariesH        int      `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"`         // 0 for off
+	CacheIgnoredFiles       bool     `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"true"`
+	ProgressUpdateIntervalS int      `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
+	SymlinksEnabled         bool     `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
+	LimitBandwidthInLan     bool     `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
+	DatabaseBlockCacheMiB   int      `xml:"databaseBlockCacheMiB" json:"databaseBlockCacheMiB" default:"0"`
+	PingTimeoutS            int      `xml:"pingTimeoutS" json:"pingTimeoutS" default:"30"`
+	PingIdleTimeS           int      `xml:"pingIdleTimeS" json:"pingIdleTimeS" default:"60"`
+}
+
+func (orig OptionsConfiguration) Copy() OptionsConfiguration {
+	c := orig
+	c.ListenAddress = make([]string, len(orig.ListenAddress))
+	copy(c.ListenAddress, orig.ListenAddress)
+	c.GlobalAnnServers = make([]string, len(orig.GlobalAnnServers))
+	copy(c.GlobalAnnServers, orig.GlobalAnnServers)
+	return c
+}
+
+type GUIConfiguration struct {
+	Enabled  bool   `xml:"enabled,attr" json:"enabled" default:"true"`
+	Address  string `xml:"address" json:"address" default:"127.0.0.1:8384"`
+	User     string `xml:"user,omitempty" json:"user"`
+	Password string `xml:"password,omitempty" json:"password"`
+	UseTLS   bool   `xml:"tls,attr" json:"useTLS"`
+	APIKey   string `xml:"apikey,omitempty" json:"apiKey"`
+}
+
+func New(myID protocol.DeviceID) Configuration {
+	var cfg Configuration
+	cfg.Version = CurrentVersion
+	cfg.OriginalVersion = CurrentVersion
+
+	setDefaults(&cfg)
+	setDefaults(&cfg.Options)
+	setDefaults(&cfg.GUI)
+
+	cfg.prepare(myID)
+
+	return cfg
+}
+
+func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
+	var cfg Configuration
+
+	setDefaults(&cfg)
+	setDefaults(&cfg.Options)
+	setDefaults(&cfg.GUI)
+
+	err := xml.NewDecoder(r).Decode(&cfg)
+	cfg.OriginalVersion = cfg.Version
+
+	cfg.prepare(myID)
+	return cfg, err
+}
+
+func (cfg *Configuration) WriteXML(w io.Writer) error {
+	e := xml.NewEncoder(w)
+	e.Indent("", "    ")
+	err := e.Encode(cfg)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write([]byte("\n"))
+	return err
+}
+
+func (cfg *Configuration) prepare(myID protocol.DeviceID) {
+	fillNilSlices(&cfg.Options)
+
+	// Initialize an empty slices
+	if cfg.Folders == nil {
+		cfg.Folders = []FolderConfiguration{}
+	}
+	if cfg.IgnoredDevices == nil {
+		cfg.IgnoredDevices = []protocol.DeviceID{}
+	}
+
+	// Check for missing, bad or duplicate folder ID:s
+	var seenFolders = map[string]*FolderConfiguration{}
+	for i := range cfg.Folders {
+		folder := &cfg.Folders[i]
+
+		if len(folder.RawPath) == 0 {
+			folder.Invalid = "no directory configured"
+			continue
+		}
+
+		// The reason it's done like this:
+		// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
+		// C:\somedir  ->  C:\somedir\    ->  C:\somedir
+		// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
+		// This way in the tests, we get away without OS specific separators
+		// in the test configs.
+		folder.RawPath = filepath.Dir(folder.RawPath + string(filepath.Separator))
+
+		if folder.ID == "" {
+			folder.ID = "default"
+		}
+
+		if folder.RescanIntervalS > MaxRescanIntervalS {
+			folder.RescanIntervalS = MaxRescanIntervalS
+		} else if folder.RescanIntervalS < 0 {
+			folder.RescanIntervalS = 0
+		}
+
+		if seen, ok := seenFolders[folder.ID]; ok {
+			l.Warnf("Multiple folders with ID %q; disabling", folder.ID)
+			seen.Invalid = "duplicate folder ID"
+			folder.Invalid = "duplicate folder ID"
+		} else {
+			seenFolders[folder.ID] = folder
+		}
+	}
+
+	if cfg.Version < OldestHandledVersion {
+		l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
+	}
+
+	// Upgrade configuration versions as appropriate
+	if cfg.Version <= 5 {
+		convertV5V6(cfg)
+	}
+	if cfg.Version == 6 {
+		convertV6V7(cfg)
+	}
+	if cfg.Version == 7 {
+		convertV7V8(cfg)
+	}
+	if cfg.Version == 8 {
+		convertV8V9(cfg)
+	}
+	if cfg.Version == 9 {
+		convertV9V10(cfg)
+	}
+
+	// Hash old cleartext passwords
+	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
+		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
+		if err != nil {
+			l.Warnln("bcrypting password:", err)
+		} else {
+			cfg.GUI.Password = string(hash)
+		}
+	}
+
+	// Build a list of available devices
+	existingDevices := make(map[protocol.DeviceID]bool)
+	for _, device := range cfg.Devices {
+		existingDevices[device.DeviceID] = true
+	}
+
+	// Ensure this device is present in the config
+	if !existingDevices[myID] {
+		myName, _ := os.Hostname()
+		cfg.Devices = append(cfg.Devices, DeviceConfiguration{
+			DeviceID: myID,
+			Name:     myName,
+		})
+		existingDevices[myID] = true
+	}
+
+	sort.Sort(DeviceConfigurationList(cfg.Devices))
+	// Ensure that any loose devices are not present in the wrong places
+	// Ensure that there are no duplicate devices
+	// Ensure that puller settings are sane
+	for i := range cfg.Folders {
+		cfg.Folders[i].Devices = ensureDevicePresent(cfg.Folders[i].Devices, myID)
+		cfg.Folders[i].Devices = ensureExistingDevices(cfg.Folders[i].Devices, existingDevices)
+		cfg.Folders[i].Devices = ensureNoDuplicates(cfg.Folders[i].Devices)
+		if cfg.Folders[i].Copiers == 0 {
+			cfg.Folders[i].Copiers = 1
+		}
+		if cfg.Folders[i].Pullers == 0 {
+			cfg.Folders[i].Pullers = 16
+		}
+		sort.Sort(FolderDeviceConfigurationList(cfg.Folders[i].Devices))
+	}
+
+	// An empty address list is equivalent to a single "dynamic" entry
+	for i := range cfg.Devices {
+		n := &cfg.Devices[i]
+		if len(n.Addresses) == 0 || len(n.Addresses) == 1 && n.Addresses[0] == "" {
+			n.Addresses = []string{"dynamic"}
+		}
+	}
+
+	// Very short reconnection intervals are annoying
+	if cfg.Options.ReconnectIntervalS < 5 {
+		cfg.Options.ReconnectIntervalS = 5
+	}
+
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+	cfg.Options.GlobalAnnServers = uniqueStrings(cfg.Options.GlobalAnnServers)
+
+	if cfg.GUI.APIKey == "" {
+		cfg.GUI.APIKey = randomString(32)
+	}
+}
+
+// ChangeRequiresRestart returns true if updating the configuration requires a
+// complete restart.
+func ChangeRequiresRestart(from, to Configuration) bool {
+	// Adding, removing or changing folders requires restart
+	if !reflect.DeepEqual(from.Folders, to.Folders) {
+		return true
+	}
+
+	// Removing a device requres restart
+	toDevs := make(map[protocol.DeviceID]bool, len(from.Devices))
+	for _, dev := range to.Devices {
+		toDevs[dev.DeviceID] = true
+	}
+	for _, dev := range from.Devices {
+		if _, ok := toDevs[dev.DeviceID]; !ok {
+			return true
+		}
+	}
+
+	// Changing usage reporting to on or off does not require a restart.
+	to.Options.URAccepted = from.Options.URAccepted
+	to.Options.URUniqueID = from.Options.URUniqueID
+
+	// All of the generic options require restart
+	if !reflect.DeepEqual(from.Options, to.Options) || !reflect.DeepEqual(from.GUI, to.GUI) {
+		return true
+	}
+
+	return false
+}
+
+func convertV9V10(cfg *Configuration) {
+	// Enable auto normalization on existing folders.
+	for i := range cfg.Folders {
+		cfg.Folders[i].AutoNormalize = true
+	}
+	cfg.Version = 10
+}
+
+func convertV8V9(cfg *Configuration) {
+	// Compression is interpreted and serialized differently, but no enforced
+	// changes. Still need a new version number since the compression stuff
+	// isn't understandable by earlier versions.
+	cfg.Version = 9
+}
+
+func convertV7V8(cfg *Configuration) {
+	// Add IPv6 announce server
+	if len(cfg.Options.GlobalAnnServers) == 1 && cfg.Options.GlobalAnnServers[0] == "udp4://announce.syncthing.net:22026" {
+		cfg.Options.GlobalAnnServers = append(cfg.Options.GlobalAnnServers, "udp6://announce-v6.syncthing.net:22026")
+	}
+
+	cfg.Version = 8
+}
+
+func convertV6V7(cfg *Configuration) {
+	// Migrate announce server addresses to the new URL based format
+	for i := range cfg.Options.GlobalAnnServers {
+		cfg.Options.GlobalAnnServers[i] = "udp4://" + cfg.Options.GlobalAnnServers[i]
+	}
+
+	cfg.Version = 7
+}
+
+func convertV5V6(cfg *Configuration) {
+	// Added ".stfolder" file at folder roots to identify mount issues
+	// Doesn't affect the config itself, but uses config migrations to identify
+	// the migration point.
+	for _, folder := range Wrap("", *cfg).Folders() {
+		// Best attempt, if it fails, it fails, the user will have to fix
+		// it up manually, as the repo will not get started.
+		folder.CreateMarker()
+	}
+
+	cfg.Version = 6
+}
+
+func setDefaults(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case string:
+				f.SetString(v)
+
+			case int:
+				i, err := strconv.ParseInt(v, 10, 64)
+				if err != nil {
+					return err
+				}
+				f.SetInt(i)
+
+			case bool:
+				f.SetBool(v == "true")
+
+			case []string:
+				// We don't do anything with string slices here. Any default
+				// we set will be appended to by the XML decoder, so we fill
+				// those after decoding.
+
+			default:
+				panic(f.Type())
+			}
+		}
+	}
+	return nil
+}
+
+// fillNilSlices sets default value on slices that are still nil.
+func fillNilSlices(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case []string:
+				if f.IsNil() {
+					// Treat the default as a comma separated slice
+					vs := strings.Split(v, ",")
+					for i := range vs {
+						vs[i] = strings.TrimSpace(vs[i])
+					}
+
+					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), len(vs), len(vs))
+					for i, v := range vs {
+						rv.Index(i).SetString(v)
+					}
+					f.Set(rv)
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func uniqueStrings(ss []string) []string {
+	var m = make(map[string]bool, len(ss))
+	for _, s := range ss {
+		m[strings.Trim(s, " ")] = true
+	}
+
+	var us = make([]string, 0, len(m))
+	for k := range m {
+		us = append(us, k)
+	}
+
+	sort.Strings(us)
+
+	return us
+}
+
+func ensureDevicePresent(devices []FolderDeviceConfiguration, myID protocol.DeviceID) []FolderDeviceConfiguration {
+	for _, device := range devices {
+		if device.DeviceID.Equals(myID) {
+			return devices
+		}
+	}
+
+	devices = append(devices, FolderDeviceConfiguration{
+		DeviceID: myID,
+	})
+
+	return devices
+}
+
+func ensureExistingDevices(devices []FolderDeviceConfiguration, existingDevices map[protocol.DeviceID]bool) []FolderDeviceConfiguration {
+	count := len(devices)
+	i := 0
+loop:
+	for i < count {
+		if _, ok := existingDevices[devices[i].DeviceID]; !ok {
+			devices[i] = devices[count-1]
+			count--
+			continue loop
+		}
+		i++
+	}
+	return devices[0:count]
+}
+
+func ensureNoDuplicates(devices []FolderDeviceConfiguration) []FolderDeviceConfiguration {
+	count := len(devices)
+	i := 0
+	seenDevices := make(map[protocol.DeviceID]bool)
+loop:
+	for i < count {
+		id := devices[i].DeviceID
+		if _, ok := seenDevices[id]; ok {
+			devices[i] = devices[count-1]
+			count--
+			continue loop
+		}
+		seenDevices[id] = true
+		i++
+	}
+	return devices[0:count]
+}
+
+type DeviceConfigurationList []DeviceConfiguration
+
+func (l DeviceConfigurationList) Less(a, b int) bool {
+	return l[a].DeviceID.Compare(l[b].DeviceID) == -1
+}
+func (l DeviceConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l DeviceConfigurationList) Len() int {
+	return len(l)
+}
+
+type FolderDeviceConfigurationList []FolderDeviceConfiguration
+
+func (l FolderDeviceConfigurationList) Less(a, b int) bool {
+	return l[a].DeviceID.Compare(l[b].DeviceID) == -1
+}
+func (l FolderDeviceConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l FolderDeviceConfigurationList) Len() int {
+	return len(l)
+}
+
+// randomCharset contains the characters that can make up a randomString().
+const randomCharset = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
+
+// randomString returns a string of random characters (taken from
+// randomCharset) of the specified length.
+func randomString(l int) string {
+	bs := make([]byte, l)
+	for i := range bs {
+		bs[i] = randomCharset[rand.Intn(len(randomCharset))]
+	}
+	return string(bs)
+}
+
+type PullOrder int
+
+const (
+	OrderRandom PullOrder = iota // default is random
+	OrderAlphabetic
+	OrderSmallestFirst
+	OrderLargestFirst
+	OrderOldestFirst
+	OrderNewestFirst
+)
+
+func (o PullOrder) String() string {
+	switch o {
+	case OrderRandom:
+		return "random"
+	case OrderAlphabetic:
+		return "alphabetic"
+	case OrderSmallestFirst:
+		return "smallestFirst"
+	case OrderLargestFirst:
+		return "largestFirst"
+	case OrderOldestFirst:
+		return "oldestFirst"
+	case OrderNewestFirst:
+		return "newestFirst"
+	default:
+		return "unknown"
+	}
+}
+
+func (o PullOrder) MarshalText() ([]byte, error) {
+	return []byte(o.String()), nil
+}
+
+func (o *PullOrder) UnmarshalText(bs []byte) error {
+	switch string(bs) {
+	case "random":
+		*o = OrderRandom
+	case "alphabetic":
+		*o = OrderAlphabetic
+	case "smallestFirst":
+		*o = OrderSmallestFirst
+	case "largestFirst":
+		*o = OrderLargestFirst
+	case "oldestFirst":
+		*o = OrderOldestFirst
+	case "newestFirst":
+		*o = OrderNewestFirst
+	default:
+		*o = OrderRandom
+	}
+	return nil
+}

lib/config/config_test.go

@@ -0,0 +1,621 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/syncthing/protocol"
+)
+
+var device1, device2, device3, device4 protocol.DeviceID
+
+func init() {
+	device1, _ = protocol.DeviceIDFromString("AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ")
+	device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
+	device3, _ = protocol.DeviceIDFromString("LGFPDIT-7SKNNJL-VJZA4FC-7QNCRKA-CE753K7-2BW5QDK-2FOZ7FR-FEP57QJ")
+	device4, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
+}
+
+func TestDefaultValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:           []string{"0.0.0.0:22000"},
+		GlobalAnnServers:        []string{"udp4://announce.syncthing.net:22026", "udp6://announce-v6.syncthing.net:22026"},
+		GlobalAnnEnabled:        true,
+		LocalAnnEnabled:         true,
+		LocalAnnPort:            21025,
+		LocalAnnMCAddr:          "[ff32::5222]:21026",
+		MaxSendKbps:             0,
+		MaxRecvKbps:             0,
+		ReconnectIntervalS:      60,
+		StartBrowser:            true,
+		UPnPEnabled:             true,
+		UPnPLeaseM:              60,
+		UPnPRenewalM:            30,
+		UPnPTimeoutS:            10,
+		RestartOnWakeup:         true,
+		AutoUpgradeIntervalH:    12,
+		KeepTemporariesH:        24,
+		CacheIgnoredFiles:       true,
+		ProgressUpdateIntervalS: 5,
+		SymlinksEnabled:         true,
+		LimitBandwidthInLan:     false,
+		DatabaseBlockCacheMiB:   0,
+		PingTimeoutS:            30,
+		PingIdleTimeS:           60,
+	}
+
+	cfg := New(device1)
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestDeviceConfig(t *testing.T) {
+	for i := OldestHandledVersion; i <= CurrentVersion; i++ {
+		os.Remove("testdata/.stfolder")
+		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = os.Stat("testdata/.stfolder")
+		if i < 6 && err != nil {
+			t.Fatal(err)
+		} else if i >= 6 && err == nil {
+			t.Fatal("Unexpected file")
+		}
+
+		cfg := wr.cfg
+
+		expectedFolders := []FolderConfiguration{
+			{
+				ID:              "test",
+				RawPath:         "testdata",
+				Devices:         []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
+				ReadOnly:        true,
+				RescanIntervalS: 600,
+				Copiers:         1,
+				Pullers:         16,
+				Hashers:         0,
+				AutoNormalize:   true,
+			},
+		}
+		expectedDevices := []DeviceConfiguration{
+			{
+				DeviceID:    device1,
+				Name:        "node one",
+				Addresses:   []string{"a"},
+				Compression: protocol.CompressMetadata,
+			},
+			{
+				DeviceID:    device4,
+				Name:        "node two",
+				Addresses:   []string{"b"},
+				Compression: protocol.CompressMetadata,
+			},
+		}
+		expectedDeviceIDs := []protocol.DeviceID{device1, device4}
+
+		if cfg.Version != CurrentVersion {
+			t.Errorf("%d: Incorrect version %d != %d", i, cfg.Version, CurrentVersion)
+		}
+		if !reflect.DeepEqual(cfg.Folders, expectedFolders) {
+			t.Errorf("%d: Incorrect Folders\n  A: %#v\n  E: %#v", i, cfg.Folders, expectedFolders)
+		}
+		if !reflect.DeepEqual(cfg.Devices, expectedDevices) {
+			t.Errorf("%d: Incorrect Devices\n  A: %#v\n  E: %#v", i, cfg.Devices, expectedDevices)
+		}
+		if !reflect.DeepEqual(cfg.Folders[0].DeviceIDs(), expectedDeviceIDs) {
+			t.Errorf("%d: Incorrect DeviceIDs\n  A: %#v\n  E: %#v", i, cfg.Folders[0].DeviceIDs(), expectedDeviceIDs)
+		}
+	}
+}
+
+func TestNoListenAddress(t *testing.T) {
+	cfg, err := Load("testdata/nolistenaddress.xml", device1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := []string{""}
+	actual := cfg.Options().ListenAddress
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Unexpected ListenAddress %#v", actual)
+	}
+}
+
+func TestOverriddenValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:           []string{":23000"},
+		GlobalAnnServers:        []string{"udp4://syncthing.nym.se:22026"},
+		GlobalAnnEnabled:        false,
+		LocalAnnEnabled:         false,
+		LocalAnnPort:            42123,
+		LocalAnnMCAddr:          "quux:3232",
+		MaxSendKbps:             1234,
+		MaxRecvKbps:             2341,
+		ReconnectIntervalS:      6000,
+		StartBrowser:            false,
+		UPnPEnabled:             false,
+		UPnPLeaseM:              90,
+		UPnPRenewalM:            15,
+		UPnPTimeoutS:            15,
+		RestartOnWakeup:         false,
+		AutoUpgradeIntervalH:    24,
+		KeepTemporariesH:        48,
+		CacheIgnoredFiles:       false,
+		ProgressUpdateIntervalS: 10,
+		SymlinksEnabled:         false,
+		LimitBandwidthInLan:     true,
+		DatabaseBlockCacheMiB:   42,
+		PingTimeoutS:            60,
+		PingIdleTimeS:           120,
+	}
+
+	cfg, err := Load("testdata/overridenvalues.xml", device1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options(), expected) {
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options())
+	}
+}
+
+func TestDeviceAddressesDynamic(t *testing.T) {
+	name, _ := os.Hostname()
+	expected := map[protocol.DeviceID]DeviceConfiguration{
+		device1: {
+			DeviceID:  device1,
+			Addresses: []string{"dynamic"},
+		},
+		device2: {
+			DeviceID:  device2,
+			Addresses: []string{"dynamic"},
+		},
+		device3: {
+			DeviceID:  device3,
+			Addresses: []string{"dynamic"},
+		},
+		device4: {
+			DeviceID:    device4,
+			Name:        name, // Set when auto created
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressMetadata,
+		},
+	}
+
+	cfg, err := Load("testdata/deviceaddressesdynamic.xml", device4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	actual := cfg.Devices()
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Devices differ;\n  E: %#v\n  A: %#v", expected, actual)
+	}
+}
+
+func TestDeviceCompression(t *testing.T) {
+	name, _ := os.Hostname()
+	expected := map[protocol.DeviceID]DeviceConfiguration{
+		device1: {
+			DeviceID:    device1,
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressMetadata,
+		},
+		device2: {
+			DeviceID:    device2,
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressMetadata,
+		},
+		device3: {
+			DeviceID:    device3,
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressNever,
+		},
+		device4: {
+			DeviceID:    device4,
+			Name:        name, // Set when auto created
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressMetadata,
+		},
+	}
+
+	cfg, err := Load("testdata/devicecompression.xml", device4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	actual := cfg.Devices()
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Devices differ;\n  E: %#v\n  A: %#v", expected, actual)
+	}
+}
+
+func TestDeviceAddressesStatic(t *testing.T) {
+	name, _ := os.Hostname()
+	expected := map[protocol.DeviceID]DeviceConfiguration{
+		device1: {
+			DeviceID:  device1,
+			Addresses: []string{"192.0.2.1", "192.0.2.2"},
+		},
+		device2: {
+			DeviceID:  device2,
+			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
+		},
+		device3: {
+			DeviceID:  device3,
+			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
+		},
+		device4: {
+			DeviceID:    device4,
+			Name:        name, // Set when auto created
+			Addresses:   []string{"dynamic"},
+			Compression: protocol.CompressMetadata,
+		},
+	}
+
+	cfg, err := Load("testdata/deviceaddressesstatic.xml", device4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	actual := cfg.Devices()
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("Devices differ;\n  E: %#v\n  A: %#v", expected, actual)
+	}
+}
+
+func TestVersioningConfig(t *testing.T) {
+	cfg, err := Load("testdata/versioningconfig.xml", device4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	vc := cfg.Folders()["test"].Versioning
+	if vc.Type != "simple" {
+		t.Errorf(`vc.Type %q != "simple"`, vc.Type)
+	}
+	if l := len(vc.Params); l != 2 {
+		t.Errorf("len(vc.Params) %d != 2", l)
+	}
+
+	expected := map[string]string{
+		"foo": "bar",
+		"baz": "quux",
+	}
+	if !reflect.DeepEqual(vc.Params, expected) {
+		t.Errorf("vc.Params differ;\n  E: %#v\n  A: %#v", expected, vc.Params)
+	}
+}
+
+func TestIssue1262(t *testing.T) {
+	cfg, err := Load("testdata/issue-1262.xml", device4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual := cfg.Folders()["test"].RawPath
+	expected := "e:"
+	if runtime.GOOS == "windows" {
+		expected = `e:\`
+	}
+
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+}
+
+func TestIssue1750(t *testing.T) {
+	cfg, err := Load("testdata/issue-1750.xml", device4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if cfg.Options().ListenAddress[0] != ":23000" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[0], ":23000")
+	}
+
+	if cfg.Options().ListenAddress[1] != ":23001" {
+		t.Errorf("%q != %q", cfg.Options().ListenAddress[1], ":23001")
+	}
+
+	if cfg.Options().GlobalAnnServers[0] != "udp4://syncthing.nym.se:22026" {
+		t.Errorf("%q != %q", cfg.Options().GlobalAnnServers[0], "udp4://syncthing.nym.se:22026")
+	}
+
+	if cfg.Options().GlobalAnnServers[1] != "udp4://syncthing.nym.se:22027" {
+		t.Errorf("%q != %q", cfg.Options().GlobalAnnServers[1], "udp4://syncthing.nym.se:22027")
+	}
+}
+
+func TestWindowsPaths(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip("Not useful on non-Windows")
+		return
+	}
+
+	folder := FolderConfiguration{
+		RawPath: `e:\`,
+	}
+
+	expected := `\\?\e:\`
+	actual := folder.Path()
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+
+	folder.RawPath = `\\192.0.2.22\network\share`
+	expected = folder.RawPath
+	actual = folder.Path()
+	if actual != expected {
+		t.Errorf("%q != %q", actual, expected)
+	}
+
+	folder.RawPath = `relative\path`
+	expected = folder.RawPath
+	actual = folder.Path()
+	if actual == expected || !strings.HasPrefix(actual, "\\\\?\\") {
+		t.Errorf("%q == %q, expected absolutification", actual, expected)
+	}
+}
+
+func TestFolderPath(t *testing.T) {
+	folder := FolderConfiguration{
+		RawPath: "~/tmp",
+	}
+
+	realPath := folder.Path()
+	if !filepath.IsAbs(realPath) {
+		t.Error(realPath, "should be absolute")
+	}
+	if strings.Contains(realPath, "~") {
+		t.Error(realPath, "should not contain ~")
+	}
+}
+
+func TestNewSaveLoad(t *testing.T) {
+	path := "testdata/temp.xml"
+	os.Remove(path)
+
+	exists := func(path string) bool {
+		_, err := os.Stat(path)
+		return err == nil
+	}
+
+	intCfg := New(device1)
+	cfg := Wrap(path, intCfg)
+
+	// To make the equality pass later
+	cfg.cfg.XMLName.Local = "configuration"
+
+	if exists(path) {
+		t.Error(path, "exists")
+	}
+
+	err := cfg.Save()
+	if err != nil {
+		t.Error(err)
+	}
+	if !exists(path) {
+		t.Error(path, "does not exist")
+	}
+
+	cfg2, err := Load(path, device1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Raw(), cfg2.Raw()) {
+		t.Errorf("Configs are not equal;\n  E:  %#v\n  A:  %#v", cfg.Raw(), cfg2.Raw())
+	}
+
+	os.Remove(path)
+}
+
+func TestPrepare(t *testing.T) {
+	var cfg Configuration
+
+	if cfg.Folders != nil || cfg.Devices != nil || cfg.Options.ListenAddress != nil {
+		t.Error("Expected nil")
+	}
+
+	cfg.prepare(device1)
+
+	if cfg.Folders == nil || cfg.Devices == nil || cfg.Options.ListenAddress == nil {
+		t.Error("Unexpected nil")
+	}
+}
+
+func TestRequiresRestart(t *testing.T) {
+	wr, err := Load("testdata/v6.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := wr.cfg
+
+	if ChangeRequiresRestart(cfg, cfg) {
+		t.Error("No change does not require restart")
+	}
+
+	newCfg := cfg
+	newCfg.Devices = append(newCfg.Devices, DeviceConfiguration{
+		DeviceID: device3,
+	})
+	if ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Adding a device does not require restart")
+	}
+
+	newCfg = cfg
+	newCfg.Devices = newCfg.Devices[:len(newCfg.Devices)-1]
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Removing a device requires restart")
+	}
+
+	newCfg = cfg
+	newCfg.Folders = append(newCfg.Folders, FolderConfiguration{
+		ID:      "t1",
+		RawPath: "t1",
+	})
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Adding a folder requires restart")
+	}
+
+	newCfg = cfg
+	newCfg.Folders = newCfg.Folders[:len(newCfg.Folders)-1]
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Removing a folder requires restart")
+	}
+
+	newCfg = cfg
+	newFolders := make([]FolderConfiguration, len(cfg.Folders))
+	copy(newFolders, cfg.Folders)
+	newCfg.Folders = newFolders
+	if ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("No changes done yet")
+	}
+	newCfg.Folders[0].RawPath = "different"
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Changing a folder requires restart")
+	}
+
+	newCfg = cfg
+	newDevices := make([]DeviceConfiguration, len(cfg.Devices))
+	copy(newDevices, cfg.Devices)
+	newCfg.Devices = newDevices
+	if ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("No changes done yet")
+	}
+	newCfg.Devices[0].Name = "different"
+	if ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Changing a device does not require restart")
+	}
+
+	newCfg = cfg
+	newCfg.Options.GlobalAnnEnabled = !cfg.Options.GlobalAnnEnabled
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Changing general options requires restart")
+	}
+
+	newCfg = cfg
+	newCfg.GUI.UseTLS = !cfg.GUI.UseTLS
+	if !ChangeRequiresRestart(cfg, newCfg) {
+		t.Error("Changing GUI options requires restart")
+	}
+}
+
+func TestCopy(t *testing.T) {
+	wrapper, err := Load("testdata/example.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := wrapper.Raw()
+
+	bsOrig, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	copy := cfg.Copy()
+
+	cfg.Devices[0].Addresses[0] = "wrong"
+	cfg.Folders[0].Devices[0].DeviceID = protocol.DeviceID{0, 1, 2, 3}
+	cfg.Options.ListenAddress[0] = "wrong"
+	cfg.GUI.APIKey = "wrong"
+
+	bsChanged, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bsCopy, err := json.MarshalIndent(copy, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if bytes.Compare(bsOrig, bsChanged) == 0 {
+		t.Error("Config should have changed")
+	}
+	if bytes.Compare(bsOrig, bsCopy) != 0 {
+		//ioutil.WriteFile("a", bsOrig, 0644)
+		//ioutil.WriteFile("b", bsCopy, 0644)
+		t.Error("Copy should be unchanged")
+	}
+}
+
+func TestPullOrder(t *testing.T) {
+	wrapper, err := Load("testdata/pullorder.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	folders := wrapper.Folders()
+
+	expected := []struct {
+		name  string
+		order PullOrder
+	}{
+		{"f1", OrderRandom},        // empty value, default
+		{"f2", OrderRandom},        // explicit
+		{"f3", OrderAlphabetic},    // explicit
+		{"f4", OrderRandom},        // unknown value, default
+		{"f5", OrderSmallestFirst}, // explicit
+		{"f6", OrderLargestFirst},  // explicit
+		{"f7", OrderOldestFirst},   // explicit
+		{"f8", OrderNewestFirst},   // explicit
+	}
+
+	// Verify values are deserialized correctly
+
+	for _, tc := range expected {
+		if actual := folders[tc.name].Order; actual != tc.order {
+			t.Errorf("Incorrect pull order for %q: %v != %v", tc.name, actual, tc.order)
+		}
+	}
+
+	// Serialize and deserialize again to verify it survives the transformation
+
+	buf := new(bytes.Buffer)
+	cfg := wrapper.Raw()
+	cfg.WriteXML(buf)
+
+	t.Logf("%s", buf.Bytes())
+
+	cfg, err = ReadXML(buf, device1)
+	wrapper = Wrap("testdata/pullorder.xml", cfg)
+	folders = wrapper.Folders()
+
+	for _, tc := range expected {
+		if actual := folders[tc.name].Order; actual != tc.order {
+			t.Errorf("Incorrect pull order for %q: %v != %v", tc.name, actual, tc.order)
+		}
+	}
+}
+
+func TestLargeRescanInterval(t *testing.T) {
+	wrapper, err := Load("testdata/largeinterval.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if wrapper.Folders()["l1"].RescanIntervalS != MaxRescanIntervalS {
+		t.Error("too large rescan interval should be maxed out")
+	}
+	if wrapper.Folders()["l2"].RescanIntervalS != 0 {
+		t.Error("negative rescan interval should become zero")
+	}
+}

lib/config/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

lib/config/testdata/deviceaddressesdynamic.xml

@@ -0,0 +1,10 @@
+<configuration version="10">
+    <device id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
+        <address></address>
+    </device>
+    <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
+    </device>
+    <device id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
+        <address>dynamic</address>
+    </device>
+</configuration>

lib/config/testdata/deviceaddressesstatic.xml

@@ -0,0 +1,14 @@
+<configuration version="3">
+    <device id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
+        <address>192.0.2.1</address>
+        <address>192.0.2.2</address>
+    </device>
+    <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
+        <address>192.0.2.3:6070</address>
+        <address>[2001:db8::42]:4242</address>
+    </device>
+    <device id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
+        <address>[2001:db8::44]:4444</address>
+        <address>192.0.2.4:6090</address>
+    </device>
+</configuration>

lib/config/testdata/devicecompression.xml

@@ -0,0 +1,8 @@
+<configuration version="5">
+    <device id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" compression="true">
+    </device>
+    <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA" compression="metadata">
+    </device>
+    <device id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q" compression="false">
+    </device>
+</configuration>

lib/config/testdata/example.xml

@@ -0,0 +1,50 @@
+<configuration version="10">
+    <folder id="default" path="~/Sync" ro="false" rescanIntervalS="60" ignorePerms="false">
+        <device id="GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <device id="ZJCXQL7-M3NP4IC-4KQ7WFU-3NANYUX-AD74QRL-Q5LJ7BH-72KYZHK-GHTAOAK"></device>
+        <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
+        <copiers>1</copiers>
+        <pullers>16</pullers>
+        <hashers>0</hashers>
+    </folder>
+    <device id="GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY" name="win7" compression="metadata" introducer="false">
+        <address>dynamic</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="jborg-mbp" compression="metadata" introducer="false">
+        <address>dynamic</address>
+    </device>
+    <device id="ZJCXQL7-M3NP4IC-4KQ7WFU-3NANYUX-AD74QRL-Q5LJ7BH-72KYZHK-GHTAOAK" name="anto-syncer" compression="never" introducer="false">
+        <address>dynamic</address>
+    </device>
+    <gui enabled="true" tls="false">
+        <address>0.0.0.0:8080</address>
+        <apikey>136020D511BF136020D511BF136020D511BF</apikey>
+    </gui>
+    <options>
+        <listenAddress>0.0.0.0:22000</listenAddress>
+        <globalAnnounceServer>udp4://announce.syncthing.net:22026</globalAnnounceServer>
+        <globalAnnounceServer>udp6://announce-v6.syncthing.net:22026</globalAnnounceServer>
+        <globalAnnounceEnabled>true</globalAnnounceEnabled>
+        <localAnnounceEnabled>true</localAnnounceEnabled>
+        <localAnnouncePort>21025</localAnnouncePort>
+        <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
+        <maxSendKbps>0</maxSendKbps>
+        <maxRecvKbps>0</maxRecvKbps>
+        <reconnectionIntervalS>60</reconnectionIntervalS>
+        <startBrowser>false</startBrowser>
+        <upnpEnabled>true</upnpEnabled>
+        <upnpLeaseMinutes>0</upnpLeaseMinutes>
+        <upnpRenewalMinutes>30</upnpRenewalMinutes>
+        <urAccepted>-1</urAccepted>
+        <urUniqueID></urUniqueID>
+        <restartOnWakeup>true</restartOnWakeup>
+        <autoUpgradeIntervalH>0</autoUpgradeIntervalH>
+        <keepTemporariesH>24</keepTemporariesH>
+        <cacheIgnoredFiles>true</cacheIgnoredFiles>
+        <progressUpdateIntervalS>5</progressUpdateIntervalS>
+        <symlinksEnabled>true</symlinksEnabled>
+        <limitBandwidthInLan>false</limitBandwidthInLan>
+    </options>
+</configuration>

lib/config/testdata/issue-1262.xml

@@ -0,0 +1,4 @@
+<configuration version="7">
+    <folder id="test" path="e:" ro="true" ignorePerms="false" rescanIntervalS="600">
+    </folder>
+</configuration>

lib/config/testdata/issue-1750.xml

@@ -0,0 +1,8 @@
+<configuration version="9">
+    <options>
+        <listenAddress> :23000</listenAddress>
+        <listenAddress>     :23001    </listenAddress>
+        <globalAnnounceServer> udp4://syncthing.nym.se:22026</globalAnnounceServer>
+        <globalAnnounceServer>   udp4://syncthing.nym.se:22027  </globalAnnounceServer>
+    </options>
+</configuration>

lib/config/testdata/largeinterval.xml

@@ -0,0 +1,4 @@
+<configuration version="10">
+    <folder id="l1" path="~/Sync" rescanIntervalS="60000000000"></folder>
+    <folder id="l2" path="~/Sync" rescanIntervalS="-1"></folder>
+</configuration>

lib/config/testdata/nolistenaddress.xml

@@ -0,0 +1,5 @@
+<configuration version="1">
+    <options>
+        <listenAddress></listenAddress>
+    </options>
+</configuration>

lib/config/testdata/overridenvalues.xml

@@ -0,0 +1,30 @@
+<configuration version="2">
+    <options>
+        <listenAddress>:23000</listenAddress>
+        <allowDelete>false</allowDelete>
+        <globalAnnounceServer>syncthing.nym.se:22026</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <localAnnouncePort>42123</localAnnouncePort>
+        <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
+        <parallelRequests>32</parallelRequests>
+        <maxSendKbps>1234</maxSendKbps>
+        <maxRecvKbps>2341</maxRecvKbps>
+        <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <startBrowser>false</startBrowser>
+        <upnpEnabled>false</upnpEnabled>
+        <upnpLeaseMinutes>90</upnpLeaseMinutes>
+        <upnpRenewalMinutes>15</upnpRenewalMinutes>
+        <upnpTimeoutSeconds>15</upnpTimeoutSeconds>
+        <restartOnWakeup>false</restartOnWakeup>
+        <autoUpgradeIntervalH>24</autoUpgradeIntervalH>
+        <keepTemporariesH>48</keepTemporariesH>
+        <cacheIgnoredFiles>false</cacheIgnoredFiles>
+        <progressUpdateIntervalS>10</progressUpdateIntervalS>
+        <symlinksEnabled>false</symlinksEnabled>
+        <limitBandwidthInLan>true</limitBandwidthInLan>
+        <databaseBlockCacheMiB>42</databaseBlockCacheMiB>
+        <pingTimeoutS>60</pingTimeoutS>
+        <pingIdleTimeS>120</pingIdleTimeS>
+    </options>
+</configuration>

lib/config/testdata/pullorder.xml

@@ -0,0 +1,25 @@
+<configuration version="10">
+    <folder id="f1" directory="testdata/">
+    </folder>
+    <folder id="f2" directory="testdata/">
+        <order>random</order>
+    </folder>
+    <folder id="f3" directory="testdata/">
+        <order>alphabetic</order>
+    </folder>
+    <folder id="f4" directory="testdata/">
+        <order>whatever</order>
+    </folder>
+    <folder id="f5" directory="testdata/">
+        <order>smallestFirst</order>
+    </folder>
+    <folder id="f6" directory="testdata/">
+        <order>largestFirst</order>
+    </folder>
+    <folder id="f7" directory="testdata/">
+        <order>oldestFirst</order>
+    </folder>
+    <folder id="f8" directory="testdata/">
+        <order>newestFirst</order>
+    </folder>
+</configuration>

lib/config/testdata/v10.xml

@@ -0,0 +1,12 @@
+<configuration version="10">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/v5.xml

@@ -0,0 +1,12 @@
+<configuration version="5">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/v6.xml

@@ -0,0 +1,12 @@
+<configuration version="6">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/v7.xml

@@ -0,0 +1,12 @@
+<configuration version="7">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/v8.xml

@@ -0,0 +1,12 @@
+<configuration version="8">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/v9.xml

@@ -0,0 +1,12 @@
+<configuration version="9">
+    <folder id="test" path="testdata" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>b</address>
+    </device>
+</configuration>

lib/config/testdata/versioningconfig.xml

@@ -0,0 +1,8 @@
+<configuration version="10">
+    <folder id="test" directory="testdata/" ro="true">
+        <versioning type="simple">
+            <param key="foo" val="bar"/>
+            <param key="baz" val="quux"/>
+        </versioning>
+    </folder>
+</configuration>

lib/config/wrapper.go

@@ -0,0 +1,300 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"os"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+// The Committer interface is implemented by objects that need to know about
+// or have a say in configuration changes.
+//
+// When the configuration is about to be changed, VerifyConfiguration() is
+// called for each subscribing object, with the old and new configuration. A
+// nil error is returned if the new configuration is acceptable (i.e. does not
+// contain any errors that would prevent it from being a valid config).
+// Otherwise an error describing the problem is returned.
+//
+// If any subscriber returns an error from VerifyConfiguration(), the
+// configuration change is not committed and an error is returned to whoever
+// tried to commit the broken config.
+//
+// If all verification calls returns nil, CommitConfiguration() is called for
+// each subscribing object. The callee returns true if the new configuration
+// has been successfully applied, otherwise false. Any Commit() call returning
+// false will result in a "restart needed" respone to the API/user. Note that
+// the new configuration will still have been applied by those who were
+// capable of doing so.
+type Committer interface {
+	VerifyConfiguration(from, to Configuration) error
+	CommitConfiguration(from, to Configuration) (handled bool)
+	String() string
+}
+
+type CommitResponse struct {
+	ValidationError error
+	RequiresRestart bool
+}
+
+var ResponseNoRestart = CommitResponse{
+	ValidationError: nil,
+	RequiresRestart: false,
+}
+
+// A wrapper around a Configuration that manages loads, saves and published
+// notifications of changes to registered Handlers
+
+type Wrapper struct {
+	cfg  Configuration
+	path string
+
+	deviceMap map[protocol.DeviceID]DeviceConfiguration
+	folderMap map[string]FolderConfiguration
+	replaces  chan Configuration
+	mut       sync.Mutex
+
+	subs []Committer
+	sMut sync.Mutex
+}
+
+// Wrap wraps an existing Configuration structure and ties it to a file on
+// disk.
+func Wrap(path string, cfg Configuration) *Wrapper {
+	w := &Wrapper{
+		cfg:  cfg,
+		path: path,
+		mut:  sync.NewMutex(),
+		sMut: sync.NewMutex(),
+	}
+	w.replaces = make(chan Configuration)
+	return w
+}
+
+// Load loads an existing file on disk and returns a new configuration
+// wrapper.
+func Load(path string, myID protocol.DeviceID) (*Wrapper, error) {
+	fd, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	cfg, err := ReadXML(fd, myID)
+	if err != nil {
+		return nil, err
+	}
+
+	return Wrap(path, cfg), nil
+}
+
+// Stop stops the Serve() loop. Set and Replace operations will panic after a
+// Stop.
+func (w *Wrapper) Stop() {
+	close(w.replaces)
+}
+
+// Subscribe registers the given handler to be called on any future
+// configuration changes.
+func (w *Wrapper) Subscribe(c Committer) {
+	w.sMut.Lock()
+	w.subs = append(w.subs, c)
+	w.sMut.Unlock()
+}
+
+// Raw returns the currently wrapped Configuration object.
+func (w *Wrapper) Raw() Configuration {
+	return w.cfg
+}
+
+// Replace swaps the current configuration object for the given one.
+func (w *Wrapper) Replace(cfg Configuration) CommitResponse {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	return w.replaceLocked(cfg)
+}
+
+func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
+	from := w.cfg
+
+	for _, sub := range w.subs {
+		if debug {
+			l.Debugln(sub, "verifying configuration")
+		}
+		if err := sub.VerifyConfiguration(from, to); err != nil {
+			if debug {
+				l.Debugln(sub, "rejected config:", err)
+			}
+			return CommitResponse{
+				ValidationError: err,
+			}
+		}
+	}
+
+	allOk := true
+	for _, sub := range w.subs {
+		if debug {
+			l.Debugln(sub, "committing configuration")
+		}
+		ok := sub.CommitConfiguration(from, to)
+		if !ok {
+			if debug {
+				l.Debugln(sub, "requires restart")
+			}
+			allOk = false
+		}
+	}
+
+	w.cfg = to
+	w.deviceMap = nil
+	w.folderMap = nil
+
+	return CommitResponse{
+		RequiresRestart: !allOk,
+	}
+}
+
+// Devices returns a map of devices. Device structures should not be changed,
+// other than for the purpose of updating via SetDevice().
+func (w *Wrapper) Devices() map[protocol.DeviceID]DeviceConfiguration {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	if w.deviceMap == nil {
+		w.deviceMap = make(map[protocol.DeviceID]DeviceConfiguration, len(w.cfg.Devices))
+		for _, dev := range w.cfg.Devices {
+			w.deviceMap[dev.DeviceID] = dev
+		}
+	}
+	return w.deviceMap
+}
+
+// SetDevice adds a new device to the configuration, or overwrites an existing
+// device with the same ID.
+func (w *Wrapper) SetDevice(dev DeviceConfiguration) CommitResponse {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+
+	newCfg := w.cfg.Copy()
+	replaced := false
+	for i := range newCfg.Devices {
+		if newCfg.Devices[i].DeviceID == dev.DeviceID {
+			newCfg.Devices[i] = dev
+			replaced = true
+			break
+		}
+	}
+	if !replaced {
+		newCfg.Devices = append(w.cfg.Devices, dev)
+	}
+
+	return w.replaceLocked(newCfg)
+}
+
+// Folders returns a map of folders. Folder structures should not be changed,
+// other than for the purpose of updating via SetFolder().
+func (w *Wrapper) Folders() map[string]FolderConfiguration {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	if w.folderMap == nil {
+		w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders))
+		for _, fld := range w.cfg.Folders {
+			w.folderMap[fld.ID] = fld
+		}
+	}
+	return w.folderMap
+}
+
+// SetFolder adds a new folder to the configuration, or overwrites an existing
+// folder with the same ID.
+func (w *Wrapper) SetFolder(fld FolderConfiguration) CommitResponse {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+
+	newCfg := w.cfg.Copy()
+	replaced := false
+	for i := range newCfg.Folders {
+		if newCfg.Folders[i].ID == fld.ID {
+			newCfg.Folders[i] = fld
+			replaced = true
+			break
+		}
+	}
+	if !replaced {
+		newCfg.Folders = append(w.cfg.Folders, fld)
+	}
+
+	return w.replaceLocked(newCfg)
+}
+
+// Options returns the current options configuration object.
+func (w *Wrapper) Options() OptionsConfiguration {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	return w.cfg.Options
+}
+
+// SetOptions replaces the current options configuration object.
+func (w *Wrapper) SetOptions(opts OptionsConfiguration) CommitResponse {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	newCfg := w.cfg.Copy()
+	newCfg.Options = opts
+	return w.replaceLocked(newCfg)
+}
+
+// GUI returns the current GUI configuration object.
+func (w *Wrapper) GUI() GUIConfiguration {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	return w.cfg.GUI
+}
+
+// SetGUI replaces the current GUI configuration object.
+func (w *Wrapper) SetGUI(gui GUIConfiguration) CommitResponse {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	newCfg := w.cfg.Copy()
+	newCfg.GUI = gui
+	return w.replaceLocked(newCfg)
+}
+
+// IgnoredDevice returns whether or not connection attempts from the given
+// device should be silently ignored.
+func (w *Wrapper) IgnoredDevice(id protocol.DeviceID) bool {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+	for _, device := range w.cfg.IgnoredDevices {
+		if device == id {
+			return true
+		}
+	}
+	return false
+}
+
+// Save writes the configuration to disk, and generates a ConfigSaved event.
+func (w *Wrapper) Save() error {
+	fd, err := osutil.CreateAtomic(w.path, 0600)
+	if err != nil {
+		return err
+	}
+
+	if err := w.cfg.WriteXML(fd); err != nil {
+		fd.Close()
+		return err
+	}
+
+	if err := fd.Close(); err != nil {
+		return err
+	}
+
+	events.Default.Log(events.ConfigSaved, w.cfg)
+	return nil
+}

lib/db/.gitignore

@@ -0,0 +1 @@
+testdata/*.db

lib/db/blockmap.go

@@ -0,0 +1,228 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package db provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+package db
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"sort"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var blockFinder *BlockFinder
+
+type BlockMap struct {
+	db     *leveldb.DB
+	folder string
+}
+
+func NewBlockMap(db *leveldb.DB, folder string) *BlockMap {
+	return &BlockMap{
+		db:     db,
+		folder: folder,
+	}
+}
+
+// Add files to the block map, ignoring any deleted or invalid files.
+func (m *BlockMap) Add(files []protocol.FileInfo) error {
+	batch := new(leveldb.Batch)
+	buf := make([]byte, 4)
+	for _, file := range files {
+		if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() {
+			continue
+		}
+
+		for i, block := range file.Blocks {
+			binary.BigEndian.PutUint32(buf, uint32(i))
+			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+		}
+	}
+	return m.db.Write(batch, nil)
+}
+
+// Update block map state, removing any deleted or invalid files.
+func (m *BlockMap) Update(files []protocol.FileInfo) error {
+	batch := new(leveldb.Batch)
+	buf := make([]byte, 4)
+	for _, file := range files {
+		if file.IsDirectory() {
+			continue
+		}
+
+		if file.IsDeleted() || file.IsInvalid() {
+			for _, block := range file.Blocks {
+				batch.Delete(m.blockKey(block.Hash, file.Name))
+			}
+			continue
+		}
+
+		for i, block := range file.Blocks {
+			binary.BigEndian.PutUint32(buf, uint32(i))
+			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+		}
+	}
+	return m.db.Write(batch, nil)
+}
+
+// Discard block map state, removing the given files
+func (m *BlockMap) Discard(files []protocol.FileInfo) error {
+	batch := new(leveldb.Batch)
+	for _, file := range files {
+		for _, block := range file.Blocks {
+			batch.Delete(m.blockKey(block.Hash, file.Name))
+		}
+	}
+	return m.db.Write(batch, nil)
+}
+
+// Drop block map, removing all entries related to this block map from the db.
+func (m *BlockMap) Drop() error {
+	batch := new(leveldb.Batch)
+	iter := m.db.NewIterator(util.BytesPrefix(m.blockKey(nil, "")[:1+64]), nil)
+	defer iter.Release()
+	for iter.Next() {
+		batch.Delete(iter.Key())
+	}
+	if iter.Error() != nil {
+		return iter.Error()
+	}
+	return m.db.Write(batch, nil)
+}
+
+func (m *BlockMap) blockKey(hash []byte, file string) []byte {
+	return toBlockKey(hash, m.folder, file)
+}
+
+type BlockFinder struct {
+	db      *leveldb.DB
+	folders []string
+	mut     sync.RWMutex
+}
+
+func NewBlockFinder(db *leveldb.DB, cfg *config.Wrapper) *BlockFinder {
+	if blockFinder != nil {
+		return blockFinder
+	}
+
+	f := &BlockFinder{
+		db:  db,
+		mut: sync.NewRWMutex(),
+	}
+
+	f.CommitConfiguration(config.Configuration{}, cfg.Raw())
+	cfg.Subscribe(f)
+
+	return f
+}
+
+// VerifyConfiguration implementes the config.Committer interface
+func (f *BlockFinder) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+// CommitConfiguration implementes the config.Committer interface
+func (f *BlockFinder) CommitConfiguration(from, to config.Configuration) bool {
+	folders := make([]string, len(to.Folders))
+	for i, folder := range to.Folders {
+		folders[i] = folder.ID
+	}
+
+	sort.Strings(folders)
+
+	f.mut.Lock()
+	f.folders = folders
+	f.mut.Unlock()
+
+	return true
+}
+
+func (f *BlockFinder) String() string {
+	return fmt.Sprintf("BlockFinder@%p", f)
+}
+
+// Iterate takes an iterator function which iterates over all matching blocks
+// for the given hash. The iterator function has to return either true (if
+// they are happy with the block) or false to continue iterating for whatever
+// reason. The iterator finally returns the result, whether or not a
+// satisfying block was eventually found.
+func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, int32) bool) bool {
+	f.mut.RLock()
+	folders := f.folders
+	f.mut.RUnlock()
+	for _, folder := range folders {
+		key := toBlockKey(hash, folder, "")
+		iter := f.db.NewIterator(util.BytesPrefix(key), nil)
+		defer iter.Release()
+
+		for iter.Next() && iter.Error() == nil {
+			folder, file := fromBlockKey(iter.Key())
+			index := int32(binary.BigEndian.Uint32(iter.Value()))
+			if iterFn(folder, osutil.NativeFilename(file), index) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Fix repairs incorrect blockmap entries, removing the old entry and
+// replacing it with a new entry for the given block
+func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []byte) error {
+	buf := make([]byte, 4)
+	binary.BigEndian.PutUint32(buf, uint32(index))
+
+	batch := new(leveldb.Batch)
+	batch.Delete(toBlockKey(oldHash, folder, file))
+	batch.Put(toBlockKey(newHash, folder, file), buf)
+	return f.db.Write(batch, nil)
+}
+
+// m.blockKey returns a byte slice encoding the following information:
+//	   keyTypeBlock (1 byte)
+//	   folder (64 bytes)
+//	   block hash (32 bytes)
+//	   file name (variable size)
+func toBlockKey(hash []byte, folder, file string) []byte {
+	o := make([]byte, 1+64+32+len(file))
+	o[0] = KeyTypeBlock
+	copy(o[1:], []byte(folder))
+	copy(o[1+64:], []byte(hash))
+	copy(o[1+64+32:], []byte(file))
+	return o
+}
+
+func fromBlockKey(data []byte) (string, string) {
+	if len(data) < 1+64+32+1 {
+		panic("Incorrect key length")
+	}
+	if data[0] != KeyTypeBlock {
+		panic("Incorrect key type")
+	}
+
+	file := string(data[1+64+32:])
+
+	slice := data[1 : 1+64]
+	izero := bytes.IndexByte(slice, 0)
+	if izero > -1 {
+		return string(slice[:izero]), file
+	}
+	return string(slice), file
+}

lib/db/blockmap_test.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"testing"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/config"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func genBlocks(n int) []protocol.BlockInfo {
+	b := make([]protocol.BlockInfo, n)
+	for i := range b {
+		h := make([]byte, 32)
+		for j := range h {
+			h[j] = byte(i + j)
+		}
+		b[i].Size = int32(i)
+		b[i].Hash = h
+	}
+	return b
+}
+
+var f1, f2, f3 protocol.FileInfo
+
+func init() {
+	blocks := genBlocks(30)
+
+	f1 = protocol.FileInfo{
+		Name:   "f1",
+		Blocks: blocks[:10],
+	}
+
+	f2 = protocol.FileInfo{
+		Name:   "f2",
+		Blocks: blocks[10:20],
+	}
+
+	f3 = protocol.FileInfo{
+		Name:   "f3",
+		Blocks: blocks[20:],
+	}
+}
+
+func setup() (*leveldb.DB, *BlockFinder) {
+	// Setup
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wrapper := config.Wrap("", config.Configuration{})
+	wrapper.SetFolder(config.FolderConfiguration{
+		ID: "folder1",
+	})
+	wrapper.SetFolder(config.FolderConfiguration{
+		ID: "folder2",
+	})
+
+	return db, NewBlockFinder(db, wrapper)
+}
+
+func dbEmpty(db *leveldb.DB) bool {
+	iter := db.NewIterator(nil, nil)
+	defer iter.Release()
+	if iter.Next() {
+		return false
+	}
+	return true
+}
+
+func TestBlockMapAddUpdateWipe(t *testing.T) {
+	db, f := setup()
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	m := NewBlockMap(db, "folder1")
+
+	f3.Flags |= protocol.FlagDirectory
+
+	err := m.Add([]protocol.FileInfo{f1, f2, f3})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		if folder != "folder1" || file != "f1" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		if folder != "folder1" || file != "f2" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		t.Fatal("Unexpected block")
+		return true
+	})
+
+	f3.Flags = f1.Flags
+	f1.Flags |= protocol.FlagDeleted
+	f2.Flags |= protocol.FlagInvalid
+
+	// Should remove
+	err = m.Update([]protocol.FileInfo{f1, f2, f3})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		t.Fatal("Unexpected block")
+		return false
+	})
+
+	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		t.Fatal("Unexpected block")
+		return false
+	})
+
+	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		if folder != "folder1" || file != "f3" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	err = m.Drop()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	// Should not add
+	err = m.Add([]protocol.FileInfo{f1, f2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	f1.Flags = 0
+	f2.Flags = 0
+	f3.Flags = 0
+}
+
+func TestBlockFinderLookup(t *testing.T) {
+	db, f := setup()
+
+	m1 := NewBlockMap(db, "folder1")
+	m2 := NewBlockMap(db, "folder2")
+
+	err := m1.Add([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = m2.Add([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	counter := 0
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		counter++
+		switch counter {
+		case 1:
+			if folder != "folder1" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		case 2:
+			if folder != "folder2" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		default:
+			t.Fatal("Unexpected block")
+		}
+		return false
+	})
+	if counter != 2 {
+		t.Fatal("Incorrect count", counter)
+	}
+
+	f1.Flags |= protocol.FlagDeleted
+
+	err = m1.Update([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	counter = 0
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index int32) bool {
+		counter++
+		switch counter {
+		case 1:
+			if folder != "folder2" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		default:
+			t.Fatal("Unexpected block")
+		}
+		return false
+	})
+	if counter != 1 {
+		t.Fatal("Incorrect count")
+	}
+
+	f1.Flags = 0
+}
+
+func TestBlockFinderFix(t *testing.T) {
+	db, f := setup()
+
+	iterFn := func(folder, file string, index int32) bool {
+		return true
+	}
+
+	m := NewBlockMap(db, "folder1")
+	err := m.Add([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !f.Iterate(f1.Blocks[0].Hash, iterFn) {
+		t.Fatal("Block not found")
+	}
+
+	err = f.Fix("folder1", f1.Name, 0, f1.Blocks[0].Hash, f2.Blocks[0].Hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if f.Iterate(f1.Blocks[0].Hash, iterFn) {
+		t.Fatal("Unexpected block")
+	}
+
+	if !f.Iterate(f2.Blocks[0].Hash, iterFn) {
+		t.Fatal("Block not found")
+	}
+}

lib/db/concurrency_test.go

@@ -0,0 +1,236 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore // this  is a really tedious test for an old issue
+
+package db_test
+
+import (
+	"crypto/rand"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var keys [][]byte
+
+func init() {
+	for i := 0; i < nItems; i++ {
+		keys = append(keys, randomData(1))
+	}
+}
+
+const nItems = 10000
+
+func randomData(prefix byte) []byte {
+	data := make([]byte, 1+32+64+32)
+	_, err := rand.Reader.Read(data)
+	if err != nil {
+		panic(err)
+	}
+	return append([]byte{prefix}, data...)
+}
+
+func setItems(db *leveldb.DB) error {
+	batch := new(leveldb.Batch)
+	for _, k1 := range keys {
+		k2 := randomData(2)
+		// k2 -> data
+		batch.Put(k2, randomData(42))
+		// k1 -> k2
+		batch.Put(k1, k2)
+	}
+	if testing.Verbose() {
+		log.Printf("batch write (set) %p", batch)
+	}
+	return db.Write(batch, nil)
+}
+
+func clearItems(db *leveldb.DB) error {
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		return err
+	}
+	defer snap.Release()
+
+	// Iterate over k2
+
+	it := snap.NewIterator(util.BytesPrefix([]byte{1}), nil)
+	defer it.Release()
+
+	batch := new(leveldb.Batch)
+	for it.Next() {
+		k1 := it.Key()
+		k2 := it.Value()
+
+		// k2 should exist
+		_, err := snap.Get(k2, nil)
+		if err != nil {
+			return err
+		}
+
+		// Delete the k1 => k2 mapping first
+		batch.Delete(k1)
+		// Then the k2 => data mapping
+		batch.Delete(k2)
+	}
+	if testing.Verbose() {
+		log.Printf("batch write (clear) %p", batch)
+	}
+	return db.Write(batch, nil)
+}
+
+func scanItems(db *leveldb.DB) error {
+	snap, err := db.GetSnapshot()
+	if testing.Verbose() {
+		log.Printf("snap create %p", snap)
+	}
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if testing.Verbose() {
+			log.Printf("snap release %p", snap)
+		}
+		snap.Release()
+	}()
+
+	// Iterate from the start of k2 space to the end
+	it := snap.NewIterator(util.BytesPrefix([]byte{1}), nil)
+	defer it.Release()
+
+	i := 0
+	for it.Next() {
+		// k2 => k1 => data
+		k1 := it.Key()
+		k2 := it.Value()
+		_, err := snap.Get(k2, nil)
+		if err != nil {
+			log.Printf("k1: %x", k1)
+			log.Printf("k2: %x (missing)", k2)
+			return err
+		}
+		i++
+	}
+	if testing.Verbose() {
+		log.Println("scanned", i)
+	}
+	return nil
+}
+
+func TestConcurrentSetClear(t *testing.T) {
+	if testing.Short() {
+		return
+	}
+
+	dur := 30 * time.Second
+	t0 := time.Now()
+	wg := sync.NewWaitGroup()
+
+	os.RemoveAll("testdata/concurrent-set-clear.db")
+	db, err := leveldb.OpenFile("testdata/concurrent-set-clear.db", &opt.Options{OpenFilesCacheCapacity: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll("testdata/concurrent-set-clear.db")
+
+	errChan := make(chan error, 3)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for time.Since(t0) < dur {
+			if err := setItems(db); err != nil {
+				errChan <- err
+				return
+			}
+			if err := clearItems(db); err != nil {
+				errChan <- err
+				return
+			}
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for time.Since(t0) < dur {
+			if err := scanItems(db); err != nil {
+				errChan <- err
+				return
+			}
+		}
+	}()
+
+	go func() {
+		wg.Wait()
+		errChan <- nil
+	}()
+
+	err = <-errChan
+	if err != nil {
+		t.Error(err)
+	}
+	db.Close()
+}
+
+func TestConcurrentSetOnly(t *testing.T) {
+	if testing.Short() {
+		return
+	}
+
+	dur := 30 * time.Second
+	t0 := time.Now()
+	wg := sync.NewWaitGroup()
+
+	os.RemoveAll("testdata/concurrent-set-only.db")
+	db, err := leveldb.OpenFile("testdata/concurrent-set-only.db", &opt.Options{OpenFilesCacheCapacity: 10})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll("testdata/concurrent-set-only.db")
+
+	errChan := make(chan error, 3)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for time.Since(t0) < dur {
+			if err := setItems(db); err != nil {
+				errChan <- err
+				return
+			}
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for time.Since(t0) < dur {
+			if err := scanItems(db); err != nil {
+				errChan <- err
+				return
+			}
+		}
+	}()
+
+	go func() {
+		wg.Wait()
+		errChan <- nil
+	}()
+
+	err = <-errChan
+	if err != nil {
+		t.Error(err)
+	}
+}

lib/db/debug.go

@@ -0,0 +1,20 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug   = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
+	debugDB = strings.Contains(os.Getenv("STTRACE"), "db") || os.Getenv("STTRACE") == "all"
+	l       = logger.DefaultLogger
+)

lib/db/leveldb_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestDeviceKey(t *testing.T) {
+	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
+	dev := []byte("device67890123456789012345678901")
+	name := []byte("name")
+
+	key := deviceKey(fld, dev, name)
+
+	fld2 := deviceKeyFolder(key)
+	if bytes.Compare(fld2, fld) != 0 {
+		t.Errorf("wrong folder %q != %q", fld2, fld)
+	}
+	dev2 := deviceKeyDevice(key)
+	if bytes.Compare(dev2, dev) != 0 {
+		t.Errorf("wrong device %q != %q", dev2, dev)
+	}
+	name2 := deviceKeyName(key)
+	if bytes.Compare(name2, name) != 0 {
+		t.Errorf("wrong name %q != %q", name2, name)
+	}
+}
+
+func TestGlobalKey(t *testing.T) {
+	fld := []byte("folder6789012345678901234567890123456789012345678901234567890123")
+	name := []byte("name")
+
+	key := globalKey(fld, name)
+
+	fld2 := globalKeyFolder(key)
+	if bytes.Compare(fld2, fld) != 0 {
+		t.Errorf("wrong folder %q != %q", fld2, fld)
+	}
+	name2 := globalKeyName(key)
+	if bytes.Compare(name2, name) != 0 {
+		t.Errorf("wrong name %q != %q", name2, name)
+	}
+}

lib/db/leveldb_xdr.go

@@ -0,0 +1,167 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package db
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+fileVersion Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                       Vector Structure                        \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Length of device                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                   device (variable length)                    \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct fileVersion {
+	Vector version;
+	opaque device<>;
+}
+
+*/
+
+func (o fileVersion) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o fileVersion) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o fileVersion) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o fileVersion) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o fileVersion) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	_, err := o.version.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
+	xw.WriteBytes(o.device)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *fileVersion) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *fileVersion) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *fileVersion) DecodeXDRFrom(xr *xdr.Reader) error {
+	(&o.version).DecodeXDRFrom(xr)
+	o.device = xr.ReadBytes()
+	return xr.Error()
+}
+
+/*
+
+versionList Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Number of versions                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\              Zero or more fileVersion Structures              \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct versionList {
+	fileVersion versions<>;
+}
+
+*/
+
+func (o versionList) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o versionList) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o versionList) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o versionList) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o versionList) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(uint32(len(o.versions)))
+	for i := range o.versions {
+		_, err := o.versions[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *versionList) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *versionList) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *versionList) DecodeXDRFrom(xr *xdr.Reader) error {
+	_versionsSize := int(xr.ReadUint32())
+	if _versionsSize < 0 {
+		return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
+	}
+	o.versions = make([]fileVersion, _versionsSize)
+	for i := range o.versions {
+		(&o.versions[i]).DecodeXDRFrom(xr)
+	}
+	return xr.Error()
+}

lib/db/namespaced.go

@@ -0,0 +1,159 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"encoding/binary"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// NamespacedKV is a simple key-value store using a specific namespace within
+// a leveldb.
+type NamespacedKV struct {
+	db     *leveldb.DB
+	prefix []byte
+}
+
+// NewNamespacedKV returns a new NamespacedKV that lives in the namespace
+// specified by the prefix.
+func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV {
+	return &NamespacedKV{
+		db:     db,
+		prefix: []byte(prefix),
+	}
+}
+
+// Reset removes all entries in this namespace.
+func (n *NamespacedKV) Reset() {
+	it := n.db.NewIterator(util.BytesPrefix(n.prefix), nil)
+	defer it.Release()
+	batch := new(leveldb.Batch)
+	for it.Next() {
+		batch.Delete(it.Key())
+		if batch.Len() > batchFlushSize {
+			if err := n.db.Write(batch, nil); err != nil {
+				panic(err)
+			}
+			batch.Reset()
+		}
+	}
+	if batch.Len() > 0 {
+		if err := n.db.Write(batch, nil); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// PutInt64 stores a new int64. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutInt64(key string, val int64) {
+	keyBs := append(n.prefix, []byte(key)...)
+	var valBs [8]byte
+	binary.BigEndian.PutUint64(valBs[:], uint64(val))
+	n.db.Put(keyBs, valBs[:], nil)
+}
+
+// Int64 returns the stored value interpreted as an int64 and a boolean that
+// is false if no value was stored at the key.
+func (n *NamespacedKV) Int64(key string) (int64, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return 0, false
+	}
+	val := binary.BigEndian.Uint64(valBs)
+	return int64(val), true
+}
+
+// PutTime stores a new time.Time. Any existing value (even if of another
+// type) is overwritten.
+func (n *NamespacedKV) PutTime(key string, val time.Time) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, _ := val.MarshalBinary() // never returns an error
+	n.db.Put(keyBs, valBs, nil)
+}
+
+// Time returns the stored value interpreted as a time.Time and a boolean
+// that is false if no value was stored at the key.
+func (n NamespacedKV) Time(key string) (time.Time, bool) {
+	var t time.Time
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return t, false
+	}
+	err = t.UnmarshalBinary(valBs)
+	return t, err == nil
+}
+
+// PutString stores a new string. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutString(key, val string) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Put(keyBs, []byte(val), nil)
+}
+
+// String returns the stored value interpreted as a string and a boolean that
+// is false if no value was stored at the key.
+func (n NamespacedKV) String(key string) (string, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return "", false
+	}
+	return string(valBs), true
+}
+
+// PutBytes stores a new byte slice. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutBytes(key string, val []byte) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Put(keyBs, val, nil)
+}
+
+// Bytes returns the stored value as a raw byte slice and a boolean that
+// is false if no value was stored at the key.
+func (n NamespacedKV) Bytes(key string) ([]byte, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return nil, false
+	}
+	return valBs, true
+}
+
+// PutBool stores a new boolean. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutBool(key string, val bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	if val {
+		n.db.Put(keyBs, []byte{0x0}, nil)
+	} else {
+		n.db.Put(keyBs, []byte{0x1}, nil)
+	}
+}
+
+// Bool returns the stored value as a boolean and a boolean that
+// is false if no value was stored at the key.
+func (n NamespacedKV) Bool(key string) (bool, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return false, false
+	}
+	return valBs[0] == 0x0, true
+}
+
+// Delete deletes the specified key. It is allowed to delete a nonexistent
+// key.
+func (n NamespacedKV) Delete(key string) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Delete(keyBs, nil)
+}

lib/db/namespaced_test.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestNamespacedInt(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+	n2 := NewNamespacedKV(ldb, "bar")
+
+	// Key is missing to start with
+
+	if v, ok := n1.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+
+	n1.PutInt64("test", 42)
+
+	// It should now exist in n1
+
+	if v, ok := n1.Int64("test"); v != 42 || !ok {
+		t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok)
+	}
+
+	// ... but not in n2, which is in a different namespace
+
+	if v, ok := n2.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+
+	n1.Delete("test")
+
+	// It should no longer exist
+
+	if v, ok := n1.Int64("test"); v != 0 || ok {
+		t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok)
+	}
+}
+
+func TestNamespacedTime(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+
+	if v, ok := n1.Time("test"); v != (time.Time{}) || ok {
+		t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok)
+	}
+
+	now := time.Now()
+	n1.PutTime("test", now)
+
+	if v, ok := n1.Time("test"); v != now || !ok {
+		t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok)
+	}
+}
+
+func TestNamespacedString(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+
+	if v, ok := n1.String("test"); v != "" || ok {
+		t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+	}
+
+	n1.PutString("test", "yo")
+
+	if v, ok := n1.String("test"); v != "yo" || !ok {
+		t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok)
+	}
+}
+
+func TestNamespacedReset(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n1 := NewNamespacedKV(ldb, "foo")
+
+	n1.PutString("test1", "yo1")
+	n1.PutString("test2", "yo2")
+	n1.PutString("test3", "yo3")
+
+	if v, ok := n1.String("test1"); v != "yo1" || !ok {
+		t.Errorf("Incorrect return v %q != \"yo1\" || ok %v != true", v, ok)
+	}
+	if v, ok := n1.String("test2"); v != "yo2" || !ok {
+		t.Errorf("Incorrect return v %q != \"yo2\" || ok %v != true", v, ok)
+	}
+	if v, ok := n1.String("test3"); v != "yo3" || !ok {
+		t.Errorf("Incorrect return v %q != \"yo3\" || ok %v != true", v, ok)
+	}
+
+	n1.Reset()
+
+	if v, ok := n1.String("test1"); v != "" || ok {
+		t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+	}
+	if v, ok := n1.String("test2"); v != "" || ok {
+		t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+	}
+	if v, ok := n1.String("test3"); v != "" || ok {
+		t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok)
+	}
+}

lib/db/set.go

@@ -0,0 +1,237 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package db provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+package db
+
+import (
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type FileSet struct {
+	localVersion map[protocol.DeviceID]int64
+	mutex        sync.Mutex
+	folder       string
+	db           *leveldb.DB
+	blockmap     *BlockMap
+}
+
+// FileIntf is the set of methods implemented by both protocol.FileInfo and
+// protocol.FileInfoTruncated.
+type FileIntf interface {
+	Size() int64
+	IsDeleted() bool
+	IsInvalid() bool
+	IsDirectory() bool
+	IsSymlink() bool
+	HasPermissionBits() bool
+}
+
+// The Iterator is called with either a protocol.FileInfo or a
+// protocol.FileInfoTruncated (depending on the method) and returns true to
+// continue iteration, false to stop.
+type Iterator func(f FileIntf) bool
+
+func NewFileSet(folder string, db *leveldb.DB) *FileSet {
+	var s = FileSet{
+		localVersion: make(map[protocol.DeviceID]int64),
+		folder:       folder,
+		db:           db,
+		blockmap:     NewBlockMap(db, folder),
+		mutex:        sync.NewMutex(),
+	}
+
+	ldbCheckGlobals(db, []byte(folder))
+
+	var deviceID protocol.DeviceID
+	ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
+		copy(deviceID[:], device)
+		if f.LocalVersion > s.localVersion[deviceID] {
+			s.localVersion[deviceID] = f.LocalVersion
+		}
+		return true
+	})
+	if debug {
+		l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
+	}
+	clock(s.localVersion[protocol.LocalDeviceID])
+
+	return &s
+}
+
+func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
+	if debug {
+		l.Debugf("%s Replace(%v, [%d])", s.folder, device, len(fs))
+	}
+	normalizeFilenames(fs)
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs)
+	if len(fs) == 0 {
+		// Reset the local version if all files were removed.
+		s.localVersion[device] = 0
+	}
+	if device == protocol.LocalDeviceID {
+		s.blockmap.Drop()
+		s.blockmap.Add(fs)
+	}
+}
+
+func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
+	if debug {
+		l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs))
+	}
+	normalizeFilenames(fs)
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if device == protocol.LocalDeviceID {
+		discards := make([]protocol.FileInfo, 0, len(fs))
+		updates := make([]protocol.FileInfo, 0, len(fs))
+		for _, newFile := range fs {
+			existingFile, ok := ldbGet(s.db, []byte(s.folder), device[:], []byte(newFile.Name))
+			if !ok || !existingFile.Version.Equal(newFile.Version) {
+				discards = append(discards, existingFile)
+				updates = append(updates, newFile)
+			}
+		}
+		s.blockmap.Discard(discards)
+		s.blockmap.Update(updates)
+	}
+	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs); lv > s.localVersion[device] {
+		s.localVersion[device] = lv
+	}
+}
+
+func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithNeed(%v)", s.folder, device)
+	}
+	ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
+	}
+	ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithHave(%v)", s.folder, device)
+	}
+	ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
+	}
+	ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithGlobal(fn Iterator) {
+	if debug {
+		l.Debugf("%s WithGlobal()", s.folder)
+	}
+	ldbWithGlobal(s.db, []byte(s.folder), nil, false, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithGlobalTruncated(fn Iterator) {
+	if debug {
+		l.Debugf("%s WithGlobalTruncated()", s.folder)
+	}
+	ldbWithGlobal(s.db, []byte(s.folder), nil, true, nativeFileIterator(fn))
+}
+
+func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
+	if debug {
+		l.Debugf("%s WithPrefixedGlobalTruncated()", s.folder, prefix)
+	}
+	ldbWithGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn))
+}
+
+func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
+	f, ok := ldbGet(s.db, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
+	f.Name = osutil.NativeFilename(f.Name)
+	return f, ok
+}
+
+func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) {
+	fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
+	if !ok {
+		return protocol.FileInfo{}, false
+	}
+	f := fi.(protocol.FileInfo)
+	f.Name = osutil.NativeFilename(f.Name)
+	return f, true
+}
+
+func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
+	fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
+	if !ok {
+		return FileInfoTruncated{}, false
+	}
+	f := fi.(FileInfoTruncated)
+	f.Name = osutil.NativeFilename(f.Name)
+	return f, true
+}
+
+func (s *FileSet) Availability(file string) []protocol.DeviceID {
+	return ldbAvailability(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
+}
+
+func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.localVersion[device]
+}
+
+// ListFolders returns the folder IDs seen in the database.
+func ListFolders(db *leveldb.DB) []string {
+	return ldbListFolders(db)
+}
+
+// DropFolder clears out all information related to the given folder from the
+// database.
+func DropFolder(db *leveldb.DB, folder string) {
+	ldbDropFolder(db, []byte(folder))
+	bm := &BlockMap{
+		db:     db,
+		folder: folder,
+	}
+	bm.Drop()
+	NewVirtualMtimeRepo(db, folder).Drop()
+}
+
+func normalizeFilenames(fs []protocol.FileInfo) {
+	for i := range fs {
+		fs[i].Name = osutil.NormalizedFilename(fs[i].Name)
+	}
+}
+
+func nativeFileIterator(fn Iterator) Iterator {
+	return func(fi FileIntf) bool {
+		switch f := fi.(type) {
+		case protocol.FileInfo:
+			f.Name = osutil.NativeFilename(f.Name)
+			return fn(f)
+		case FileInfoTruncated:
+			f.Name = osutil.NativeFilename(f.Name)
+			return fn(f)
+		default:
+			panic("unknown interface type")
+		}
+	}
+}

lib/db/set_test.go

@@ -0,0 +1,783 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db_test
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+var remoteDevice0, remoteDevice1 protocol.DeviceID
+
+func init() {
+	remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
+	remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
+}
+
+const myID = 1
+
+func genBlocks(n int) []protocol.BlockInfo {
+	b := make([]protocol.BlockInfo, n)
+	for i := range b {
+		h := make([]byte, 32)
+		for j := range h {
+			h[j] = byte(i + j)
+		}
+		b[i].Size = int32(i)
+		b[i].Hash = h
+	}
+	return b
+}
+
+func globalList(s *db.FileSet) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithGlobal(func(fi db.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithHave(n, func(fi db.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithNeed(n, func(fi db.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+type fileList []protocol.FileInfo
+
+func (l fileList) Len() int {
+	return len(l)
+}
+
+func (l fileList) Less(a, b int) bool {
+	return l[a].Name < l[b].Name
+}
+
+func (l fileList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l fileList) String() string {
+	var b bytes.Buffer
+	b.WriteString("[]protocol.FileList{\n")
+	for _, f := range l {
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
+	}
+	b.WriteString("}")
+	return b.String()
+}
+
+func TestGlobalSet(t *testing.T) {
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+
+	local0 := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+	}
+	local1 := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+	}
+	localTot := fileList{
+		local0[0],
+		local0[1],
+		local0[2],
+		local0[3],
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+	}
+
+	remote0 := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
+	}
+	remote1 := fileList{
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
+	}
+	remoteTot := fileList{
+		remote0[0],
+		remote1[0],
+		remote0[2],
+		remote1[1],
+	}
+
+	expectedGlobal := fileList{
+		remote0[0],  // a
+		remote1[0],  // b
+		remote0[2],  // c
+		localTot[3], // d
+		remote1[1],  // e
+		localTot[4], // z
+	}
+
+	expectedLocalNeed := fileList{
+		remote1[0],
+		remote0[2],
+		remote1[1],
+	}
+
+	expectedRemoteNeed := fileList{
+		local0[3],
+	}
+
+	m.Replace(protocol.LocalDeviceID, local0)
+	m.Replace(protocol.LocalDeviceID, local1)
+	m.Replace(remoteDevice0, remote0)
+	m.Update(remoteDevice0, remote1)
+
+	g := fileList(globalList(m))
+	sort.Sort(g)
+
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
+	}
+
+	h := fileList(haveList(m, protocol.LocalDeviceID))
+	sort.Sort(h)
+
+	if fmt.Sprint(h) != fmt.Sprint(localTot) {
+		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
+	}
+
+	h = fileList(haveList(m, remoteDevice0))
+	sort.Sort(h)
+
+	if fmt.Sprint(h) != fmt.Sprint(remoteTot) {
+		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, remoteTot)
+	}
+
+	n := fileList(needList(m, protocol.LocalDeviceID))
+	sort.Sort(n)
+
+	if fmt.Sprint(n) != fmt.Sprint(expectedLocalNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedLocalNeed)
+	}
+
+	n = fileList(needList(m, remoteDevice0))
+	sort.Sort(n)
+
+	if fmt.Sprint(n) != fmt.Sprint(expectedRemoteNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
+	}
+
+	f, ok := m.Get(protocol.LocalDeviceID, "b")
+	if !ok {
+		t.Error("Unexpectedly not OK")
+	}
+	if fmt.Sprint(f) != fmt.Sprint(localTot[1]) {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
+	}
+
+	f, ok = m.Get(remoteDevice0, "b")
+	if !ok {
+		t.Error("Unexpectedly not OK")
+	}
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
+	}
+
+	f, ok = m.GetGlobal("b")
+	if !ok {
+		t.Error("Unexpectedly not OK")
+	}
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
+	}
+
+	f, ok = m.Get(protocol.LocalDeviceID, "zz")
+	if ok {
+		t.Error("Unexpectedly OK")
+	}
+	if f.Name != "" {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
+	}
+
+	f, ok = m.GetGlobal("zz")
+	if ok {
+		t.Error("Unexpectedly OK")
+	}
+	if f.Name != "" {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
+	}
+
+	av := []protocol.DeviceID{protocol.LocalDeviceID, remoteDevice0}
+	a := m.Availability("a")
+	if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
+	}
+	a = m.Availability("b")
+	if len(a) != 1 || a[0] != remoteDevice0 {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteDevice0)
+	}
+	a = m.Availability("d")
+	if len(a) != 1 || a[0] != protocol.LocalDeviceID {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalDeviceID)
+	}
+}
+
+func TestNeedWithInvalid(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := db.NewFileSet("test", ldb)
+
+	localHave := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+	}
+	remote0Have := fileList{
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+	}
+	remote1Have := fileList{
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+
+	expectedNeed := fileList{
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+	}
+
+	s.Replace(protocol.LocalDeviceID, localHave)
+	s.Replace(remoteDevice0, remote0Have)
+	s.Replace(remoteDevice1, remote1Have)
+
+	need := fileList(needList(s, protocol.LocalDeviceID))
+	sort.Sort(need)
+
+	if fmt.Sprint(need) != fmt.Sprint(expectedNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed)
+	}
+}
+
+func TestUpdateToInvalid(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := db.NewFileSet("test", ldb)
+
+	localHave := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+	}
+
+	s.Replace(protocol.LocalDeviceID, localHave)
+
+	have := fileList(haveList(s, protocol.LocalDeviceID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+
+	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
+	s.Update(protocol.LocalDeviceID, localHave[1:2])
+
+	have = fileList(haveList(s, protocol.LocalDeviceID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect after invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+}
+
+func TestInvalidAvailability(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := db.NewFileSet("test", ldb)
+
+	remote0Have := fileList{
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+	remote1Have := fileList{
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+
+	s.Replace(remoteDevice0, remote0Have)
+	s.Replace(remoteDevice1, remote1Have)
+
+	if av := s.Availability("both"); len(av) != 2 {
+		t.Error("Incorrect availability for 'both':", av)
+	}
+
+	if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
+		t.Error("Incorrect availability for 'r0only':", av)
+	}
+
+	if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
+		t.Error("Incorrect availability for 'r1only':", av)
+	}
+
+	if av := s.Availability("none"); len(av) != 0 {
+		t.Error("Incorrect availability for 'none':", av)
+	}
+}
+func Benchmark10kReplace(b *testing.B) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m := db.NewFileSet("test", ldb)
+		m.Replace(protocol.LocalDeviceID, local)
+	}
+}
+
+func Benchmark10kUpdateChg(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+	m.Replace(remoteDevice0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		for j := range local {
+			local[j].Version = local[j].Version.Update(myID)
+		}
+		b.StartTimer()
+		m.Update(protocol.LocalDeviceID, local)
+	}
+}
+
+func Benchmark10kUpdateSme(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	m := db.NewFileSet("test", ldb)
+	m.Replace(remoteDevice0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Update(protocol.LocalDeviceID, local)
+	}
+}
+
+func Benchmark10kNeed2k(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+	m.Replace(remoteDevice0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 8000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+	for i := 8000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := needList(m, protocol.LocalDeviceID)
+		if l := len(fs); l != 2000 {
+			b.Errorf("wrong length %d != 2k", l)
+		}
+	}
+}
+
+func Benchmark10kHaveFullList(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+	m.Replace(remoteDevice0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 2000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := haveList(m, protocol.LocalDeviceID)
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
+func Benchmark10kGlobal(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+	m.Replace(remoteDevice0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 2000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{ID: myID, Value: 1000}}})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: protocol.Vector{{1, 980}}})
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := globalList(m)
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
+func TestGlobalReset(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+
+	local := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	remote := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+	g := globalList(m)
+	sort.Sort(fileList(g))
+
+	if fmt.Sprint(g) != fmt.Sprint(local) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
+	}
+
+	m.Replace(remoteDevice0, remote)
+	m.Replace(remoteDevice0, nil)
+
+	g = globalList(m)
+	sort.Sort(fileList(g))
+
+	if fmt.Sprint(g) != fmt.Sprint(local) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
+	}
+}
+
+func TestNeed(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+
+	local := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	remote := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	shouldNeed := []protocol.FileInfo{
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	m.Replace(protocol.LocalDeviceID, local)
+	m.Replace(remoteDevice0, remote)
+
+	need := needList(m, protocol.LocalDeviceID)
+
+	sort.Sort(fileList(need))
+	sort.Sort(fileList(shouldNeed))
+
+	if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
+		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
+	}
+}
+
+func TestLocalVersion(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := db.NewFileSet("test", ldb)
+
+	local1 := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	local2 := []protocol.FileInfo{
+		local1[0],
+		// [1] deleted
+		local1[2],
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	m.Replace(protocol.LocalDeviceID, local1)
+	c0 := m.LocalVersion(protocol.LocalDeviceID)
+
+	m.Replace(protocol.LocalDeviceID, local2)
+	c1 := m.LocalVersion(protocol.LocalDeviceID)
+	if !(c1 > c0) {
+		t.Fatal("Local version number should have incremented")
+	}
+}
+
+func TestListDropFolder(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s0 := db.NewFileSet("test0", ldb)
+	local1 := []protocol.FileInfo{
+		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+	s0.Replace(protocol.LocalDeviceID, local1)
+
+	s1 := db.NewFileSet("test1", ldb)
+	local2 := []protocol.FileInfo{
+		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+	}
+	s1.Replace(remoteDevice0, local2)
+
+	// Check that we have both folders and their data is in the global list
+
+	expectedFolderList := []string{"test0", "test1"}
+	if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
+		t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s1", l)
+	}
+
+	// Drop one of them and check that it's gone.
+
+	db.DropFolder(ldb, "test1")
+
+	expectedFolderList = []string{"test0"}
+	if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) {
+		t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 0 {
+		t.Errorf("Incorrect global length %d != 0 for s1", l)
+	}
+}
+
+func TestGlobalNeedWithInvalid(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := db.NewFileSet("test1", ldb)
+
+	rem0 := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+	}
+	s.Replace(remoteDevice0, rem0)
+
+	rem1 := fileList{
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+	}
+	s.Replace(remoteDevice1, rem1)
+
+	total := fileList{
+		// There's a valid copy of each file, so it should be merged
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+	}
+
+	need := fileList(needList(s, protocol.LocalDeviceID))
+	if fmt.Sprint(need) != fmt.Sprint(total) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total)
+	}
+
+	global := fileList(globalList(s))
+	if fmt.Sprint(global) != fmt.Sprint(total) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", global, total)
+	}
+}
+
+func TestLongPath(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := db.NewFileSet("test", ldb)
+
+	var b bytes.Buffer
+	for i := 0; i < 100; i++ {
+		b.WriteString("012345678901234567890123456789012345678901234567890")
+	}
+	name := b.String() // 5000 characters
+
+	local := []protocol.FileInfo{
+		{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+	}
+
+	s.Replace(protocol.LocalDeviceID, local)
+
+	gf := globalList(s)
+	if l := len(gf); l != 1 {
+		t.Fatalf("Incorrect len %d != 1 for global list", l)
+	}
+	if gf[0].Name != local[0].Name {
+		t.Errorf("Incorrect long filename;\n%q !=\n%q",
+			gf[0].Name, local[0].Name)
+	}
+}

lib/db/testdata/.gitignore

@@ -0,0 +1 @@
+index.db

lib/db/truncated.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import "github.com/syncthing/protocol"
+
+type FileInfoTruncated struct {
+	protocol.FileInfo
+	ActualSize int64
+}
+
+func (f *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
+	err := f.FileInfo.UnmarshalXDR(bs)
+	f.ActualSize = f.FileInfo.Size()
+	f.FileInfo.Blocks = nil
+	return err
+}
+
+func (f FileInfoTruncated) Size() int64 {
+	return f.ActualSize
+}
+
+func BlocksToSize(num int) int64 {
+	if num < 2 {
+		return protocol.BlockSize / 2
+	}
+	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+}

lib/db/virtualmtime.go

@@ -0,0 +1,84 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// This type encapsulates a repository of mtimes for platforms where file mtimes
+// can't be set to arbitrary values.  For this to work, we need to store both
+// the mtime we tried to set (the "actual" mtime) as well as the mtime the file
+// has when we're done touching it (the "disk" mtime) so that we can tell if it
+// was changed.  So in GetMtime(), it's not sufficient that the record exists --
+// the argument must also equal the "disk" mtime in the record, otherwise it's
+// been touched locally and the "disk" mtime is actually correct.
+
+type VirtualMtimeRepo struct {
+	ns *NamespacedKV
+}
+
+func NewVirtualMtimeRepo(ldb *leveldb.DB, folder string) *VirtualMtimeRepo {
+	prefix := string(KeyTypeVirtualMtime) + folder
+
+	return &VirtualMtimeRepo{
+		ns: NewNamespacedKV(ldb, prefix),
+	}
+}
+
+func (r *VirtualMtimeRepo) UpdateMtime(path string, diskMtime, actualMtime time.Time) {
+	if debug {
+		l.Debugf("virtual mtime: storing values for path:%s disk:%v actual:%v", path, diskMtime, actualMtime)
+	}
+
+	diskBytes, _ := diskMtime.MarshalBinary()
+	actualBytes, _ := actualMtime.MarshalBinary()
+
+	data := append(diskBytes, actualBytes...)
+
+	r.ns.PutBytes(path, data)
+}
+
+func (r *VirtualMtimeRepo) GetMtime(path string, diskMtime time.Time) time.Time {
+	data, exists := r.ns.Bytes(path)
+	if !exists {
+		// Absense of debug print is significant enough in itself here
+		return diskMtime
+	}
+
+	var mtime time.Time
+	if err := mtime.UnmarshalBinary(data[:len(data)/2]); err != nil {
+		panic(fmt.Sprintf("Can't unmarshal stored mtime at path %s: %v", path, err))
+	}
+
+	if mtime.Equal(diskMtime) {
+		if err := mtime.UnmarshalBinary(data[len(data)/2:]); err != nil {
+			panic(fmt.Sprintf("Can't unmarshal stored mtime at path %s: %v", path, err))
+		}
+
+		if debug {
+			l.Debugf("virtual mtime: return %v instead of %v for path: %s", mtime, diskMtime, path)
+		}
+		return mtime
+	}
+
+	if debug {
+		l.Debugf("virtual mtime: record exists, but mismatch inDisk: %v dbDisk: %v for path: %s", diskMtime, mtime, path)
+	}
+	return diskMtime
+}
+
+func (r *VirtualMtimeRepo) DeleteMtime(path string) {
+	r.ns.Delete(path)
+}
+
+func (r *VirtualMtimeRepo) Drop() {
+	r.ns.Reset()
+}

lib/db/virtualmtime_test.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestVirtualMtimeRepo(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A few repos so we can ensure they don't pollute each other
+	repo1 := NewVirtualMtimeRepo(ldb, "folder1")
+	repo2 := NewVirtualMtimeRepo(ldb, "folder2")
+
+	// Since GetMtime() returns its argument if the key isn't found or is outdated, we need a dummy to test with.
+	dummyTime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
+
+	// Some times to test with
+	time1 := time.Date(2001, time.February, 3, 4, 5, 7, 0, time.UTC)
+	time2 := time.Date(2010, time.February, 3, 4, 5, 6, 0, time.UTC)
+
+	file1 := "file1.txt"
+
+	// Files are not present at the start
+
+	if v := repo1.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be missing (%v) from repo 1 but it's %v", dummyTime, v)
+	}
+
+	if v := repo2.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be missing (%v) from repo 2 but it's %v", dummyTime, v)
+	}
+
+	repo1.UpdateMtime(file1, time1, time2)
+
+	// Now it should return time2 only when time1 is passed as the argument
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time2) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time2, time1, v)
+	}
+
+	if v := repo1.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", dummyTime, dummyTime, v)
+	}
+
+	// repo2 shouldn't know about this file
+
+	if v := repo2.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v in repo 2 but we got %v", time1, time1, v)
+	}
+
+	repo1.DeleteMtime(file1)
+
+	// Now it should be gone
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time1, time1, v)
+	}
+
+	// Try again but with Drop()
+
+	repo1.UpdateMtime(file1, time1, time2)
+	repo1.Drop()
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time1, time1, v)
+	}
+}

lib/discover/client.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/syncthing/protocol"
+)
+
+type Factory func(*url.URL, *Announce) (Client, error)
+
+var (
+	factories                      = make(map[string]Factory)
+	DefaultErrorRetryInternval     = 60 * time.Second
+	DefaultGlobalBroadcastInterval = 1800 * time.Second
+)
+
+func Register(proto string, factory Factory) {
+	factories[proto] = factory
+}
+
+func New(addr string, pkt *Announce) (Client, error) {
+	uri, err := url.Parse(addr)
+	if err != nil {
+		return nil, err
+	}
+	factory, ok := factories[uri.Scheme]
+	if !ok {
+		return nil, fmt.Errorf("Unsupported scheme: %s", uri.Scheme)
+	}
+	client, err := factory(uri, pkt)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+type Client interface {
+	Lookup(device protocol.DeviceID) []string
+	StatusOK() bool
+	Address() string
+	Stop()
+}

lib/discover/client_test.go

@@ -0,0 +1,219 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"testing"
+
+	"github.com/syncthing/protocol"
+
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+var device protocol.DeviceID
+
+func init() {
+	device, _ = protocol.DeviceIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
+}
+
+func TestUDP4Success(t *testing.T) {
+	conn, err := net.ListenUDP("udp4", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	port := conn.LocalAddr().(*net.UDPAddr).Port
+
+	address := fmt.Sprintf("udp4://127.0.0.1:%d", port)
+	pkt := &Announce{
+		Magic: AnnouncementMagic,
+		This: Device{
+			device[:],
+			[]Address{{
+				IP:   net.IPv4(123, 123, 123, 123),
+				Port: 1234,
+			}},
+		},
+	}
+
+	client, err := New(address, pkt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	udpclient := client.(*UDPClient)
+	if udpclient.errorRetryInterval != DefaultErrorRetryInternval {
+		t.Fatal("Incorrect retry interval")
+	}
+
+	if udpclient.listenAddress.IP != nil || udpclient.listenAddress.Port != 0 {
+		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
+	}
+
+	if client.Address() != address {
+		t.Fatal("Incorrect address")
+	}
+
+	buf := make([]byte, 2048)
+
+	// First announcement
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, err = conn.Read(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Announcement verification
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
+	_, addr, err := conn.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Reply to it.
+	_, err = conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// We should get nothing else
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, err = conn.Read(buf)
+	if err == nil {
+		t.Fatal("Expected error")
+	}
+
+	// Status should be ok
+	if !client.StatusOK() {
+		t.Fatal("Wrong status")
+	}
+
+	// Do a lookup in a separate routine
+	addrs := []string{}
+	wg := sync.NewWaitGroup()
+	wg.Add(1)
+	go func() {
+		addrs = client.Lookup(device)
+		wg.Done()
+	}()
+
+	// Receive the lookup and reply
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, addr, err = conn.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	conn.WriteToUDP(pkt.MustMarshalXDR(), addr)
+
+	// Wait for the lookup to arrive, verify that the number of answers is correct
+	wg.Wait()
+
+	if len(addrs) != 1 || addrs[0] != "123.123.123.123:1234" {
+		t.Fatal("Wrong number of answers")
+	}
+
+	client.Stop()
+}
+
+func TestUDP4Failure(t *testing.T) {
+	conn, err := net.ListenUDP("udp4", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	port := conn.LocalAddr().(*net.UDPAddr).Port
+
+	address := fmt.Sprintf("udp4://127.0.0.1:%d/?listenaddress=127.0.0.1&retry=5", port)
+
+	pkt := &Announce{
+		Magic: AnnouncementMagic,
+		This: Device{
+			device[:],
+			[]Address{{
+				IP:   net.IPv4(123, 123, 123, 123),
+				Port: 1234,
+			}},
+		},
+	}
+
+	client, err := New(address, pkt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	udpclient := client.(*UDPClient)
+	if udpclient.errorRetryInterval != time.Second*5 {
+		t.Fatal("Incorrect retry interval")
+	}
+
+	if !udpclient.listenAddress.IP.Equal(net.IPv4(127, 0, 0, 1)) || udpclient.listenAddress.Port != 0 {
+		t.Fatal("Wrong listen IP or port", udpclient.listenAddress)
+	}
+
+	if client.Address() != address {
+		t.Fatal("Incorrect address")
+	}
+
+	buf := make([]byte, 2048)
+
+	// First announcement
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, err = conn.Read(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Announcement verification
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 1100))
+	_, _, err = conn.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Don't reply
+	// We should get nothing else
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, err = conn.Read(buf)
+	if err == nil {
+		t.Fatal("Expected error")
+	}
+
+	// Status should be failure
+	if client.StatusOK() {
+		t.Fatal("Wrong status")
+	}
+
+	// Do a lookup in a separate routine
+	addrs := []string{}
+	wg := sync.NewWaitGroup()
+	wg.Add(1)
+	go func() {
+		addrs = client.Lookup(device)
+		wg.Done()
+	}()
+
+	// Receive the lookup and don't reply
+	conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+	_, _, err = conn.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the lookup to timeout, verify that the number of answers is none
+	wg.Wait()
+
+	if len(addrs) != 0 {
+		t.Fatal("Wrong number of answers")
+	}
+
+	client.Stop()
+}

lib/discover/client_udp.go

@@ -0,0 +1,243 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"encoding/hex"
+	"io"
+	"net"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func init() {
+	for _, proto := range []string{"udp", "udp4", "udp6"} {
+		Register(proto, func(uri *url.URL, pkt *Announce) (Client, error) {
+			c := &UDPClient{
+				wg:  sync.NewWaitGroup(),
+				mut: sync.NewRWMutex(),
+			}
+			err := c.Start(uri, pkt)
+			if err != nil {
+				return nil, err
+			}
+			return c, nil
+		})
+	}
+}
+
+type UDPClient struct {
+	url *url.URL
+
+	id protocol.DeviceID
+
+	stop          chan struct{}
+	wg            sync.WaitGroup
+	listenAddress *net.UDPAddr
+
+	globalBroadcastInterval time.Duration
+	errorRetryInterval      time.Duration
+
+	status bool
+	mut    sync.RWMutex
+}
+
+func (d *UDPClient) Start(uri *url.URL, pkt *Announce) error {
+	d.url = uri
+	d.id = protocol.DeviceIDFromBytes(pkt.This.ID)
+	d.stop = make(chan struct{})
+
+	params := uri.Query()
+	// The address must not have a port, as otherwise both announce and lookup
+	// sockets would try to bind to the same port.
+	addr, err := net.ResolveUDPAddr(d.url.Scheme, params.Get("listenaddress")+":0")
+	if err != nil {
+		return err
+	}
+	d.listenAddress = addr
+
+	broadcastSeconds, err := strconv.ParseUint(params.Get("broadcast"), 0, 0)
+	if err != nil {
+		d.globalBroadcastInterval = DefaultGlobalBroadcastInterval
+	} else {
+		d.globalBroadcastInterval = time.Duration(broadcastSeconds) * time.Second
+	}
+
+	retrySeconds, err := strconv.ParseUint(params.Get("retry"), 0, 0)
+	if err != nil {
+		d.errorRetryInterval = DefaultErrorRetryInternval
+	} else {
+		d.errorRetryInterval = time.Duration(retrySeconds) * time.Second
+	}
+
+	d.wg.Add(1)
+	go d.broadcast(pkt.MustMarshalXDR())
+	return nil
+}
+
+func (d *UDPClient) broadcast(pkt []byte) {
+	defer d.wg.Done()
+
+	conn, err := net.ListenUDP(d.url.Scheme, d.listenAddress)
+	for err != nil {
+		if debug {
+			l.Debugf("discover %s: broadcast listen: %v; trying again in %v", d.url, err, d.errorRetryInterval)
+		}
+		select {
+		case <-d.stop:
+			return
+		case <-time.After(d.errorRetryInterval):
+		}
+		conn, err = net.ListenUDP(d.url.Scheme, d.listenAddress)
+	}
+	defer conn.Close()
+
+	remote, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
+	for err != nil {
+		if debug {
+			l.Debugf("discover %s: broadcast resolve: %v; trying again in %v", d.url, err, d.errorRetryInterval)
+		}
+		select {
+		case <-d.stop:
+			return
+		case <-time.After(d.errorRetryInterval):
+		}
+		remote, err = net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
+	}
+
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-d.stop:
+			return
+
+		case <-timer.C:
+			var ok bool
+
+			if debug {
+				l.Debugf("discover %s: broadcast: Sending self announcement to %v", d.url, remote)
+			}
+
+			_, err := conn.WriteTo(pkt, remote)
+			if err != nil {
+				if debug {
+					l.Debugf("discover %s: broadcast: Failed to send self announcement: %s", d.url, err)
+				}
+				ok = false
+			} else {
+				// Verify that the announce server responds positively for our device ID
+
+				time.Sleep(1 * time.Second)
+
+				res := d.Lookup(d.id)
+				if debug {
+					l.Debugf("discover %s: broadcast: Self-lookup returned: %v", d.url, res)
+				}
+				ok = len(res) > 0
+			}
+
+			d.mut.Lock()
+			d.status = ok
+			d.mut.Unlock()
+
+			if ok {
+				timer.Reset(d.globalBroadcastInterval)
+			} else {
+				timer.Reset(d.errorRetryInterval)
+			}
+		}
+	}
+}
+
+func (d *UDPClient) Lookup(device protocol.DeviceID) []string {
+	extIP, err := net.ResolveUDPAddr(d.url.Scheme, d.url.Host)
+	if err != nil {
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
+		}
+		return nil
+	}
+
+	conn, err := net.DialUDP(d.url.Scheme, d.listenAddress, extIP)
+	if err != nil {
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
+		}
+		return nil
+	}
+	defer conn.Close()
+
+	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
+	if err != nil {
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
+		}
+		return nil
+	}
+
+	buf := Query{QueryMagic, device[:]}.MustMarshalXDR()
+	_, err = conn.Write(buf)
+	if err != nil {
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
+		}
+		return nil
+	}
+
+	buf = make([]byte, 2048)
+	n, err := conn.Read(buf)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			// Expected if the server doesn't know about requested device ID
+			return nil
+		}
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s", d.url, device, err)
+		}
+		return nil
+	}
+
+	var pkt Announce
+	err = pkt.UnmarshalXDR(buf[:n])
+	if err != nil && err != io.EOF {
+		if debug {
+			l.Debugf("discover %s: Lookup(%s): %s\n%s", d.url, device, err, hex.Dump(buf[:n]))
+		}
+		return nil
+	}
+
+	var addrs []string
+	for _, a := range pkt.This.Addresses {
+		deviceAddr := net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
+		addrs = append(addrs, deviceAddr)
+	}
+	if debug {
+		l.Debugf("discover %s: Lookup(%s) result: %v", d.url, device, addrs)
+	}
+	return addrs
+}
+
+func (d *UDPClient) Stop() {
+	if d.stop != nil {
+		close(d.stop)
+		d.wg.Wait()
+	}
+}
+
+func (d *UDPClient) StatusOK() bool {
+	d.mut.RLock()
+	defer d.mut.RUnlock()
+	return d.status
+}
+
+func (d *UDPClient) Address() string {
+	return d.url.String()
+}

lib/discover/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

lib/discover/discover.go

@@ -0,0 +1,453 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/beacon"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type Discoverer struct {
+	myID            protocol.DeviceID
+	listenAddrs     []string
+	localBcastIntv  time.Duration
+	localBcastStart time.Time
+	cacheLifetime   time.Duration
+	negCacheCutoff  time.Duration
+	beacons         []beacon.Interface
+	extPort         uint16
+	localBcastTick  <-chan time.Time
+	forcedBcastTick chan time.Time
+
+	registryLock sync.RWMutex
+	registry     map[protocol.DeviceID][]CacheEntry
+	lastLookup   map[protocol.DeviceID]time.Time
+
+	clients []Client
+	mut     sync.RWMutex
+}
+
+type CacheEntry struct {
+	Address string
+	Seen    time.Time
+}
+
+var (
+	ErrIncorrectMagic = errors.New("incorrect magic number")
+)
+
+func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
+	return &Discoverer{
+		myID:           id,
+		listenAddrs:    addresses,
+		localBcastIntv: 30 * time.Second,
+		cacheLifetime:  5 * time.Minute,
+		negCacheCutoff: 3 * time.Minute,
+		registry:       make(map[protocol.DeviceID][]CacheEntry),
+		lastLookup:     make(map[protocol.DeviceID]time.Time),
+		registryLock:   sync.NewRWMutex(),
+		mut:            sync.NewRWMutex(),
+	}
+}
+
+func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
+	if localPort > 0 {
+		d.startLocalIPv4Broadcasts(localPort)
+	}
+
+	if len(localMCAddr) > 0 {
+		d.startLocalIPv6Multicasts(localMCAddr)
+	}
+
+	if len(d.beacons) == 0 {
+		l.Warnln("Local discovery unavailable")
+		return
+	}
+
+	d.localBcastTick = time.Tick(d.localBcastIntv)
+	d.forcedBcastTick = make(chan time.Time)
+	d.localBcastStart = time.Now()
+	go d.sendLocalAnnouncements()
+}
+
+func (d *Discoverer) startLocalIPv4Broadcasts(localPort int) {
+	bb := beacon.NewBroadcast(localPort)
+	d.beacons = append(d.beacons, bb)
+	go d.recvAnnouncements(bb)
+	bb.ServeBackground()
+}
+
+func (d *Discoverer) startLocalIPv6Multicasts(localMCAddr string) {
+	intfs, err := net.Interfaces()
+	if err != nil {
+		if debug {
+			l.Debugln("discover: interfaces:", err)
+		}
+		l.Infoln("Local discovery over IPv6 unavailable")
+		return
+	}
+
+	v6Intfs := 0
+	for _, intf := range intfs {
+		// Interface flags seem to always be 0 on Windows
+		if runtime.GOOS != "windows" && (intf.Flags&net.FlagUp == 0 || intf.Flags&net.FlagMulticast == 0) {
+			continue
+		}
+
+		mb, err := beacon.NewMulticast(localMCAddr, intf.Name)
+		if err != nil {
+			if debug {
+				l.Debugln("discover: Start local v6:", err)
+			}
+			continue
+		}
+
+		d.beacons = append(d.beacons, mb)
+		go d.recvAnnouncements(mb)
+		v6Intfs++
+	}
+
+	if v6Intfs == 0 {
+		l.Infoln("Local discovery over IPv6 unavailable")
+	}
+}
+
+func (d *Discoverer) StartGlobal(servers []string, extPort uint16) {
+	d.mut.Lock()
+	defer d.mut.Unlock()
+
+	if len(d.clients) > 0 {
+		d.stopGlobal()
+	}
+
+	d.extPort = extPort
+	pkt := d.announcementPkt()
+	wg := sync.NewWaitGroup()
+	clients := make(chan Client, len(servers))
+	for _, address := range servers {
+		wg.Add(1)
+		go func(addr string) {
+			defer wg.Done()
+			client, err := New(addr, pkt)
+			if err != nil {
+				l.Infoln("Error creating discovery client", addr, err)
+				return
+			}
+			clients <- client
+		}(address)
+	}
+
+	wg.Wait()
+	close(clients)
+
+	for client := range clients {
+		d.clients = append(d.clients, client)
+	}
+}
+
+func (d *Discoverer) StopGlobal() {
+	d.mut.Lock()
+	defer d.mut.Unlock()
+	d.stopGlobal()
+}
+
+func (d *Discoverer) stopGlobal() {
+	for _, client := range d.clients {
+		client.Stop()
+	}
+	d.clients = []Client{}
+}
+
+func (d *Discoverer) ExtAnnounceOK() map[string]bool {
+	d.mut.RLock()
+	defer d.mut.RUnlock()
+
+	ret := make(map[string]bool)
+	for _, client := range d.clients {
+		ret[client.Address()] = client.StatusOK()
+	}
+	return ret
+}
+
+func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
+	d.registryLock.RLock()
+	cached := d.filterCached(d.registry[device])
+	lastLookup := d.lastLookup[device]
+	d.registryLock.RUnlock()
+
+	d.mut.RLock()
+	defer d.mut.RUnlock()
+
+	if len(cached) > 0 {
+		// There are cached address entries.
+		addrs := make([]string, len(cached))
+		for i := range cached {
+			addrs[i] = cached[i].Address
+		}
+		return addrs
+	}
+
+	if time.Since(lastLookup) < d.negCacheCutoff {
+		// We have recently tried to lookup this address and failed. Lets
+		// chill for a while.
+		return nil
+	}
+
+	if len(d.clients) != 0 && time.Since(d.localBcastStart) > d.localBcastIntv {
+		// Only perform external lookups if we have at least one external
+		// server client and one local announcement interval has passed. This is
+		// to avoid finding local peers on their remote address at startup.
+		results := make(chan []string, len(d.clients))
+		wg := sync.NewWaitGroup()
+		for _, client := range d.clients {
+			wg.Add(1)
+			go func(c Client) {
+				defer wg.Done()
+				results <- c.Lookup(device)
+			}(client)
+		}
+
+		wg.Wait()
+		close(results)
+
+		cached := []CacheEntry{}
+		seen := make(map[string]struct{})
+		now := time.Now()
+
+		var addrs []string
+		for result := range results {
+			for _, addr := range result {
+				_, ok := seen[addr]
+				if !ok {
+					cached = append(cached, CacheEntry{
+						Address: addr,
+						Seen:    now,
+					})
+					seen[addr] = struct{}{}
+					addrs = append(addrs, addr)
+				}
+			}
+		}
+
+		d.registryLock.Lock()
+		d.registry[device] = cached
+		d.lastLookup[device] = time.Now()
+		d.registryLock.Unlock()
+
+		return addrs
+	}
+
+	return nil
+}
+
+func (d *Discoverer) Hint(device string, addrs []string) {
+	resAddrs := resolveAddrs(addrs)
+	var id protocol.DeviceID
+	id.UnmarshalText([]byte(device))
+	d.registerDevice(nil, Device{
+		Addresses: resAddrs,
+		ID:        id[:],
+	})
+}
+
+func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
+	d.registryLock.RLock()
+	devices := make(map[protocol.DeviceID][]CacheEntry, len(d.registry))
+	for device, addrs := range d.registry {
+		addrsCopy := make([]CacheEntry, len(addrs))
+		copy(addrsCopy, addrs)
+		devices[device] = addrsCopy
+	}
+	d.registryLock.RUnlock()
+	return devices
+}
+
+func (d *Discoverer) announcementPkt() *Announce {
+	var addrs []Address
+	if d.extPort != 0 {
+		addrs = []Address{{Port: d.extPort}}
+	} else {
+		for _, astr := range d.listenAddrs {
+			addr, err := net.ResolveTCPAddr("tcp", astr)
+			if err != nil {
+				l.Warnln("discover: %v: not announcing %s", err, astr)
+				continue
+			} else if debug {
+				l.Debugf("discover: resolved %s as %#v", astr, addr)
+			}
+			if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+				addrs = append(addrs, Address{Port: uint16(addr.Port)})
+			} else if bs := addr.IP.To4(); bs != nil {
+				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
+			} else if bs := addr.IP.To16(); bs != nil {
+				addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
+			}
+		}
+	}
+	return &Announce{
+		Magic: AnnouncementMagic,
+		This:  Device{d.myID[:], addrs},
+	}
+}
+
+func (d *Discoverer) sendLocalAnnouncements() {
+	var addrs = resolveAddrs(d.listenAddrs)
+
+	var pkt = Announce{
+		Magic: AnnouncementMagic,
+		This:  Device{d.myID[:], addrs},
+	}
+	msg := pkt.MustMarshalXDR()
+
+	for {
+		for _, b := range d.beacons {
+			b.Send(msg)
+		}
+
+		select {
+		case <-d.localBcastTick:
+		case <-d.forcedBcastTick:
+		}
+	}
+}
+
+func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
+	for {
+		buf, addr := b.Recv()
+
+		var pkt Announce
+		err := pkt.UnmarshalXDR(buf)
+		if err != nil && err != io.EOF {
+			if debug {
+				l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
+			}
+			continue
+		}
+
+		if debug {
+			l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		}
+
+		var newDevice bool
+		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
+			newDevice = d.registerDevice(addr, pkt.This)
+		}
+
+		if newDevice {
+			select {
+			case d.forcedBcastTick <- time.Now():
+			}
+		}
+	}
+}
+
+func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
+	var id protocol.DeviceID
+	copy(id[:], device.ID)
+
+	d.registryLock.Lock()
+	defer d.registryLock.Unlock()
+
+	current := d.filterCached(d.registry[id])
+
+	orig := current
+
+	for _, a := range device.Addresses {
+		var deviceAddr string
+		if len(a.IP) > 0 {
+			deviceAddr = net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
+		} else if addr != nil {
+			ua := addr.(*net.UDPAddr)
+			ua.Port = int(a.Port)
+			deviceAddr = ua.String()
+		}
+		for i := range current {
+			if current[i].Address == deviceAddr {
+				current[i].Seen = time.Now()
+				goto done
+			}
+		}
+		current = append(current, CacheEntry{
+			Address: deviceAddr,
+			Seen:    time.Now(),
+		})
+	done:
+	}
+
+	if debug {
+		l.Debugf("discover: Caching %s addresses: %v", id, current)
+	}
+
+	d.registry[id] = current
+
+	if len(current) > len(orig) {
+		addrs := make([]string, len(current))
+		for i := range current {
+			addrs[i] = current[i].Address
+		}
+		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
+			"device": id.String(),
+			"addrs":  addrs,
+		})
+	}
+
+	return len(current) > len(orig)
+}
+
+func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
+	for i := 0; i < len(c); {
+		if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
+			if debug {
+				l.Debugf("discover: Removing cached address %s - seen %v ago", c[i].Address, ago)
+			}
+			c[i] = c[len(c)-1]
+			c = c[:len(c)-1]
+		} else {
+			i++
+		}
+	}
+	return c
+}
+
+func addrToAddr(addr *net.TCPAddr) Address {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return Address{Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To4(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To16(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	}
+	return Address{}
+}
+
+func resolveAddrs(addrs []string) []Address {
+	var raddrs []Address
+	for _, addrStr := range addrs {
+		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr.IP) > 0 {
+			raddrs = append(raddrs, addr)
+		} else {
+			raddrs = append(raddrs, Address{Port: addr.Port})
+		}
+	}
+	return raddrs
+}

lib/discover/discover_test.go

@@ -0,0 +1,139 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"net/url"
+	"time"
+
+	"testing"
+
+	"github.com/syncthing/protocol"
+)
+
+type DummyClient struct {
+	url          *url.URL
+	lookups      []protocol.DeviceID
+	lookupRet    []string
+	stops        int
+	statusRet    bool
+	statusChecks int
+}
+
+func (c *DummyClient) Lookup(device protocol.DeviceID) []string {
+	c.lookups = append(c.lookups, device)
+	return c.lookupRet
+}
+
+func (c *DummyClient) StatusOK() bool {
+	c.statusChecks++
+	return c.statusRet
+}
+
+func (c *DummyClient) Stop() {
+	c.stops++
+}
+
+func (c *DummyClient) Address() string {
+	return c.url.String()
+}
+
+func TestGlobalDiscovery(t *testing.T) {
+	c1 := &DummyClient{
+		statusRet: false,
+		lookupRet: []string{"test.com:1234"},
+	}
+
+	c2 := &DummyClient{
+		statusRet: true,
+		lookupRet: []string{},
+	}
+
+	c3 := &DummyClient{
+		statusRet: true,
+		lookupRet: []string{"best.com:2345"},
+	}
+
+	clients := []*DummyClient{c1, c2}
+
+	Register("test1", func(uri *url.URL, pkt *Announce) (Client, error) {
+		c := clients[0]
+		clients = clients[1:]
+		c.url = uri
+		return c, nil
+	})
+
+	Register("test2", func(uri *url.URL, pkt *Announce) (Client, error) {
+		c3.url = uri
+		return c3, nil
+	})
+
+	d := NewDiscoverer(device, []string{})
+	d.localBcastStart = time.Time{}
+	servers := []string{
+		"test1://123.123.123.123:1234",
+		"test1://23.23.23.23:234",
+		"test2://234.234.234.234.2345",
+	}
+	d.StartGlobal(servers, 1234)
+
+	if len(d.clients) != 3 {
+		t.Fatal("Wrong number of clients")
+	}
+
+	status := d.ExtAnnounceOK()
+
+	for _, c := range []*DummyClient{c1, c2, c3} {
+		if status[c.url.String()] != c.statusRet || c.statusChecks != 1 {
+			t.Fatal("Wrong status")
+		}
+	}
+
+	addrs := d.Lookup(device)
+	if len(addrs) != 2 {
+		t.Fatal("Wrong number of addresses", addrs)
+	}
+
+	for _, addr := range []string{"test.com:1234", "best.com:2345"} {
+		found := false
+		for _, laddr := range addrs {
+			if laddr == addr {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatal("Couldn't find", addr)
+		}
+	}
+
+	for _, c := range []*DummyClient{c1, c2, c3} {
+		if len(c.lookups) != 1 || c.lookups[0] != device {
+			t.Fatal("Wrong lookups")
+		}
+	}
+
+	addrs = d.Lookup(device)
+	if len(addrs) != 2 {
+		t.Fatal("Wrong number of addresses", addrs)
+	}
+
+	// Answer should be cached, so number of lookups should have not increased
+	for _, c := range []*DummyClient{c1, c2, c3} {
+		if len(c.lookups) != 1 || c.lookups[0] != device {
+			t.Fatal("Wrong lookups")
+		}
+	}
+
+	d.StopGlobal()
+
+	for _, c := range []*DummyClient{c1, c2, c3} {
+		if c.stops != 1 {
+			t.Fatal("Wrong number of stops")
+		}
+	}
+}

lib/discover/doc.go

@@ -0,0 +1,8 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package discover implements the device discovery protocol.
+package discover

lib/discover/packets.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
+//go:generate genxdr -o packets_xdr.go packets.go
+
+package discover
+
+const (
+	AnnouncementMagic = 0x9D79BC39
+	QueryMagic        = 0x2CA856F5
+)
+
+type Query struct {
+	Magic    uint32
+	DeviceID []byte // max:32
+}
+
+type Announce struct {
+	Magic uint32
+	This  Device
+	Extra []Device // max:16
+}
+
+type Device struct {
+	ID        []byte    // max:32
+	Addresses []Address // max:16
+}
+
+type Address struct {
+	IP   []byte // max:16
+	Port uint16
+}

lib/discover/packets_xdr.go

@@ -0,0 +1,357 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package discover
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+Query Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Magic                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Length of Device ID                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                  Device ID (variable length)                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Query {
+	unsigned int Magic;
+	opaque DeviceID<32>;
+}
+
+*/
+
+func (o Query) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Query) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Query) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Query) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Query) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.Magic)
+	if l := len(o.DeviceID); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("DeviceID", l, 32)
+	}
+	xw.WriteBytes(o.DeviceID)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Query) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Query) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Query) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Magic = xr.ReadUint32()
+	o.DeviceID = xr.ReadBytesMax(32)
+	return xr.Error()
+}
+
+/*
+
+Announce Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Magic                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                       Device Structure                        \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Number of Extra                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Device Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Announce {
+	unsigned int Magic;
+	Device This;
+	Device Extra<16>;
+}
+
+*/
+
+func (o Announce) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Announce) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Announce) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Announce) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Announce) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.Magic)
+	_, err := o.This.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
+	if l := len(o.Extra); l > 16 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Extra", l, 16)
+	}
+	xw.WriteUint32(uint32(len(o.Extra)))
+	for i := range o.Extra {
+		_, err := o.Extra[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Announce) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Announce) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Announce) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Magic = xr.ReadUint32()
+	(&o.This).DecodeXDRFrom(xr)
+	_ExtraSize := int(xr.ReadUint32())
+	if _ExtraSize < 0 {
+		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
+	}
+	if _ExtraSize > 16 {
+		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
+	}
+	o.Extra = make([]Device, _ExtraSize)
+	for i := range o.Extra {
+		(&o.Extra[i]).DecodeXDRFrom(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Device Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of ID                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     ID (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Number of Addresses                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Address Structures                \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Device {
+	opaque ID<32>;
+	Address Addresses<16>;
+}
+
+*/
+
+func (o Device) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Device) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Device) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Device) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.ID); l > 32 {
+		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32)
+	}
+	xw.WriteBytes(o.ID)
+	if l := len(o.Addresses); l > 16 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Addresses", l, 16)
+	}
+	xw.WriteUint32(uint32(len(o.Addresses)))
+	for i := range o.Addresses {
+		_, err := o.Addresses[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Device) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Device) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.ID = xr.ReadBytesMax(32)
+	_AddressesSize := int(xr.ReadUint32())
+	if _AddressesSize < 0 {
+		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
+	}
+	if _AddressesSize > 16 {
+		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
+	}
+	o.Addresses = make([]Address, _AddressesSize)
+	for i := range o.Addresses {
+		(&o.Addresses[i]).DecodeXDRFrom(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Address Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of IP                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     IP (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |             Port              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Address {
+	opaque IP<16>;
+	unsigned int Port;
+}
+
+*/
+
+func (o Address) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o Address) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Address) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o Address) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o Address) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	if l := len(o.IP); l > 16 {
+		return xw.Tot(), xdr.ElementSizeExceeded("IP", l, 16)
+	}
+	xw.WriteBytes(o.IP)
+	xw.WriteUint16(o.Port)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Address) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Address) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *Address) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.IP = xr.ReadBytesMax(16)
+	o.Port = xr.ReadUint16()
+	return xr.Error()
+}

lib/events/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package events
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all"
+	dl    = logger.DefaultLogger
+)

lib/events/events.go

@@ -0,0 +1,274 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package events provides event subscription and polling functionality.
+package events
+
+import (
+	"errors"
+	stdsync "sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type EventType int
+
+const (
+	Ping EventType = 1 << iota
+	Starting
+	StartupComplete
+	DeviceDiscovered
+	DeviceConnected
+	DeviceDisconnected
+	DeviceRejected
+	LocalIndexUpdated
+	RemoteIndexUpdated
+	ItemStarted
+	ItemFinished
+	StateChanged
+	FolderRejected
+	ConfigSaved
+	DownloadProgress
+	FolderSummary
+	FolderCompletion
+	FolderErrors
+
+	AllEvents = (1 << iota) - 1
+)
+
+func (t EventType) String() string {
+	switch t {
+	case Ping:
+		return "Ping"
+	case Starting:
+		return "Starting"
+	case StartupComplete:
+		return "StartupComplete"
+	case DeviceDiscovered:
+		return "DeviceDiscovered"
+	case DeviceConnected:
+		return "DeviceConnected"
+	case DeviceDisconnected:
+		return "DeviceDisconnected"
+	case DeviceRejected:
+		return "DeviceRejected"
+	case LocalIndexUpdated:
+		return "LocalIndexUpdated"
+	case RemoteIndexUpdated:
+		return "RemoteIndexUpdated"
+	case ItemStarted:
+		return "ItemStarted"
+	case ItemFinished:
+		return "ItemFinished"
+	case StateChanged:
+		return "StateChanged"
+	case FolderRejected:
+		return "FolderRejected"
+	case ConfigSaved:
+		return "ConfigSaved"
+	case DownloadProgress:
+		return "DownloadProgress"
+	case FolderSummary:
+		return "FolderSummary"
+	case FolderCompletion:
+		return "FolderCompletion"
+	case FolderErrors:
+		return "FolderErrors"
+	default:
+		return "Unknown"
+	}
+}
+
+func (t EventType) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+const BufferSize = 64
+
+type Logger struct {
+	subs   map[int]*Subscription
+	nextID int
+	mutex  sync.Mutex
+}
+
+type Event struct {
+	ID   int         `json:"id"`
+	Time time.Time   `json:"time"`
+	Type EventType   `json:"type"`
+	Data interface{} `json:"data"`
+}
+
+type Subscription struct {
+	mask    EventType
+	id      int
+	events  chan Event
+	timeout *time.Timer
+}
+
+var Default = NewLogger()
+
+var (
+	ErrTimeout = errors.New("timeout")
+	ErrClosed  = errors.New("closed")
+)
+
+func NewLogger() *Logger {
+	return &Logger{
+		subs:  make(map[int]*Subscription),
+		mutex: sync.NewMutex(),
+	}
+}
+
+func (l *Logger) Log(t EventType, data interface{}) {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("log", l.nextID, t.String(), data)
+	}
+	e := Event{
+		ID:   l.nextID,
+		Time: time.Now(),
+		Type: t,
+		Data: data,
+	}
+	l.nextID++
+	for _, s := range l.subs {
+		if s.mask&t != 0 {
+			select {
+			case s.events <- e:
+			default:
+				// if s.events is not ready, drop the event
+			}
+		}
+	}
+	l.mutex.Unlock()
+}
+
+func (l *Logger) Subscribe(mask EventType) *Subscription {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("subscribe", mask)
+	}
+	s := &Subscription{
+		mask:    mask,
+		id:      l.nextID,
+		events:  make(chan Event, BufferSize),
+		timeout: time.NewTimer(0),
+	}
+	l.nextID++
+	l.subs[s.id] = s
+	l.mutex.Unlock()
+	return s
+}
+
+func (l *Logger) Unsubscribe(s *Subscription) {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("unsubscribe")
+	}
+	delete(l.subs, s.id)
+	close(s.events)
+	l.mutex.Unlock()
+}
+
+// Poll returns an event from the subscription or an error if the poll times
+// out of the event channel is closed. Poll should not be called concurrently
+// from multiple goroutines for a single subscription.
+func (s *Subscription) Poll(timeout time.Duration) (Event, error) {
+	if debug {
+		dl.Debugln("poll", timeout)
+	}
+
+	s.timeout.Reset(timeout)
+	select {
+	case e, ok := <-s.events:
+		if !ok {
+			return e, ErrClosed
+		}
+		return e, nil
+	case <-s.timeout.C:
+		return Event{}, ErrTimeout
+	}
+}
+
+func (s *Subscription) C() <-chan Event {
+	return s.events
+}
+
+type BufferedSubscription struct {
+	sub  *Subscription
+	buf  []Event
+	next int
+	cur  int
+	mut  sync.Mutex
+	cond *stdsync.Cond
+}
+
+func NewBufferedSubscription(s *Subscription, size int) *BufferedSubscription {
+	bs := &BufferedSubscription{
+		sub: s,
+		buf: make([]Event, size),
+		mut: sync.NewMutex(),
+	}
+	bs.cond = stdsync.NewCond(bs.mut)
+	go bs.pollingLoop()
+	return bs
+}
+
+func (s *BufferedSubscription) pollingLoop() {
+	for {
+		ev, err := s.sub.Poll(60 * time.Second)
+		if err == ErrTimeout {
+			continue
+		}
+		if err == ErrClosed {
+			return
+		}
+		if err != nil {
+			panic("unexpected error: " + err.Error())
+		}
+
+		s.mut.Lock()
+		s.buf[s.next] = ev
+		s.next = (s.next + 1) % len(s.buf)
+		s.cur = ev.ID
+		s.cond.Broadcast()
+		s.mut.Unlock()
+	}
+}
+
+func (s *BufferedSubscription) Since(id int, into []Event) []Event {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	for id >= s.cur {
+		s.cond.Wait()
+	}
+
+	for i := s.next; i < len(s.buf); i++ {
+		if s.buf[i].ID > id {
+			into = append(into, s.buf[i])
+		}
+	}
+	for i := 0; i < s.next; i++ {
+		if s.buf[i].ID > id {
+			into = append(into, s.buf[i])
+		}
+	}
+
+	return into
+}
+
+// Error returns a string pointer suitable for JSON marshalling errors. It
+// retains the "null on sucess" semantics, but ensures the error result is a
+// string regardless of the underlying concrete error type.
+func Error(err error) *string {
+	if err == nil {
+		return nil
+	}
+	str := err.Error()
+	return &str
+}

lib/events/events_test.go

@@ -0,0 +1,180 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package events_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/events"
+)
+
+var timeout = 100 * time.Millisecond
+
+func TestNewLogger(t *testing.T) {
+	l := events.NewLogger()
+	if l == nil {
+		t.Fatal("Unexpected nil Logger")
+	}
+}
+
+func TestSubscriber(t *testing.T) {
+	l := events.NewLogger()
+	s := l.Subscribe(0)
+	if s == nil {
+		t.Fatal("Unexpected nil Subscription")
+	}
+}
+
+func TestTimeout(t *testing.T) {
+	l := events.NewLogger()
+	s := l.Subscribe(0)
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestEventBeforeSubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	l.Log(events.DeviceConnected, "foo")
+	s := l.Subscribe(0)
+
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestEventAfterSubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.DeviceConnected, "foo")
+
+	ev, err := s.Poll(timeout)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Type != events.DeviceConnected {
+		t.Error("Incorrect event type", ev.Type)
+	}
+	switch v := ev.Data.(type) {
+	case string:
+		if v != "foo" {
+			t.Error("Incorrect Data string", v)
+		}
+	default:
+		t.Errorf("Incorrect Data type %#v", v)
+	}
+}
+
+func TestEventAfterSubscribeIgnoreMask(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.DeviceDisconnected)
+	l.Log(events.DeviceConnected, "foo")
+
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestBufferOverflow(t *testing.T) {
+	l := events.NewLogger()
+
+	_ = l.Subscribe(events.AllEvents)
+
+	t0 := time.Now()
+	for i := 0; i < events.BufferSize*2; i++ {
+		l.Log(events.DeviceConnected, "foo")
+	}
+	if time.Since(t0) > timeout {
+		t.Fatalf("Logging took too long")
+	}
+}
+
+func TestUnsubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.DeviceConnected, "foo")
+
+	_, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	l.Unsubscribe(s)
+	l.Log(events.DeviceConnected, "foo")
+
+	_, err = s.Poll(timeout)
+	if err != events.ErrClosed {
+		t.Fatal("Unexpected non-Closed error:", err)
+	}
+}
+
+func TestIDs(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.DeviceConnected, "foo")
+	l.Log(events.DeviceConnected, "bar")
+
+	ev, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Data.(string) != "foo" {
+		t.Fatal("Incorrect event:", ev)
+	}
+	id := ev.ID
+
+	ev, err = s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Data.(string) != "bar" {
+		t.Fatal("Incorrect event:", ev)
+	}
+	if !(ev.ID > id) {
+		t.Fatalf("ID not incremented (%d !> %d)", ev.ID, id)
+	}
+}
+
+func TestBufferedSub(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	bs := events.NewBufferedSubscription(s, 10*events.BufferSize)
+
+	go func() {
+		for i := 0; i < 10*events.BufferSize; i++ {
+			l.Log(events.DeviceConnected, fmt.Sprintf("event-%d", i))
+			if i%30 == 0 {
+				// Give the buffer routine time to pick up the events
+				time.Sleep(20 * time.Millisecond)
+			}
+		}
+	}()
+
+	recv := 0
+	for recv < 10*events.BufferSize {
+		evs := bs.Since(recv, nil)
+		for _, ev := range evs {
+			if ev.ID != recv+1 {
+				t.Fatalf("Incorrect ID; %d != %d", ev.ID, recv+1)
+			}
+			recv = ev.ID
+		}
+	}
+
+}

lib/fnmatch/fnmatch.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+const (
+	NoEscape = (1 << iota)
+	PathName
+	CaseFold
+)
+
+func Convert(pattern string, flags int) (*regexp.Regexp, error) {
+	any := "."
+
+	switch runtime.GOOS {
+	case "windows":
+		flags |= NoEscape | CaseFold
+		pattern = filepath.FromSlash(pattern)
+		if flags&PathName != 0 {
+			any = "[^\\\\]"
+		}
+	case "darwin":
+		flags |= CaseFold
+		fallthrough
+	default:
+		if flags&PathName != 0 {
+			any = "[^/]"
+		}
+	}
+
+	// Support case insensitive ignores. We do the loop because we may in some
+	// circumstances end up with multiple insensitivity prefixes
+	// ("(?i)(?i)foo"), which should be accepted.
+	for ignore := strings.TrimPrefix(pattern, "(?i)"); ignore != pattern; ignore = strings.TrimPrefix(pattern, "(?i)") {
+		flags |= CaseFold
+		pattern = ignore
+	}
+
+	if flags&NoEscape != 0 {
+		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+	} else {
+		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+	}
+
+	// Characters that are special in regexps but not in glob, must be
+	// escaped.
+	for _, char := range []string{".", "+", "$", "^", "(", ")", "|"} {
+		pattern = strings.Replace(pattern, char, "\\"+char, -1)
+	}
+
+	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
+	pattern = strings.Replace(pattern, "*", any+"*", -1)
+	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
+	pattern = strings.Replace(pattern, "?", any, -1)
+
+	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
+	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
+	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+
+	pattern = "^" + pattern + "$"
+	if flags&CaseFold != 0 {
+		pattern = "(?i)" + pattern
+	}
+	return regexp.Compile(pattern)
+}
+
+// Match matches the pattern against the string, with the given flags, and
+// returns true if the match is successful.
+func Match(pattern, s string, flags int) (bool, error) {
+	exp, err := Convert(pattern, flags)
+	if err != nil {
+		return false, err
+	}
+	return exp.MatchString(s), nil
+}

lib/fnmatch/fnmatch_test.go

@@ -0,0 +1,102 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+type testcase struct {
+	pat   string
+	name  string
+	flags int
+	match bool
+}
+
+var testcases = []testcase{
+	{"", "", 0, true},
+	{"*", "", 0, true},
+	{"*", "foo", 0, true},
+	{"*", "bar", 0, true},
+	{"*", "*", 0, true},
+	{"**", "f", 0, true},
+	{"**", "foo.txt", 0, true},
+	{"*.*", "foo.txt", 0, true},
+	{"foo*.txt", "foobar.txt", 0, true},
+	{"foo.txt", "foo.txt", 0, true},
+
+	{"foo.txt", "bar/foo.txt", 0, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"f?o.txt", "foo.txt", 0, true},
+	{"f?o.txt", "fooo.txt", 0, false},
+	{"f[ab]o.txt", "foo.txt", 0, false},
+	{"f[ab]o.txt", "fao.txt", 0, true},
+	{"f[ab]o.txt", "fbo.txt", 0, true},
+	{"f[ab]o.txt", "fco.txt", 0, false},
+	{"f[ab]o.txt", "fabo.txt", 0, false},
+	{"f[ab]o.txt", "f[ab]o.txt", 0, false},
+	{"f\\[ab\\]o.txt", "f[ab]o.txt", NoEscape, false},
+
+	{"*foo.txt", "bar/foo.txt", 0, true},
+	{"*foo.txt", "bar/foo.txt", PathName, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"*/foo.txt", "bar/foo.txt", PathName, true},
+	{"*/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"*/foo.txt", "bar/baz/foo.txt", PathName, false},
+	{"**/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"**/foo.txt", "bar/baz/foo.txt", PathName, true},
+
+	{"foo.txt", "foo.TXT", CaseFold, true},
+	{"(?i)foo.txt", "foo.TXT", 0, true},
+	{"(?i)(?i)foo.txt", "foo.TXT", 0, true}, // repeated prefix should be fine
+	{"(?i)**foo.txt", "/dev/tmp/foo.TXT", 0, true},
+	{"(?i)!**foo.txt", "/dev/tmp/foo.TXT", 0, false},
+
+	// These characters are literals in glob, but not in regexp.
+	{"hey$hello", "hey$hello", 0, true},
+	{"hey^hello", "hey^hello", 0, true},
+	{"hey{hello", "hey{hello", 0, true},
+	{"hey}hello", "hey}hello", 0, true},
+	{"hey(hello", "hey(hello", 0, true},
+	{"hey)hello", "hey)hello", 0, true},
+	{"hey|hello", "hey|hello", 0, true},
+	{"hey|hello", "hey|other", 0, false},
+}
+
+func TestMatch(t *testing.T) {
+	switch runtime.GOOS {
+	case "windows":
+		testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
+	case "darwin":
+		testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
+		fallthrough
+	default:
+		testcases = append(testcases, testcase{"f\\[ab\\]o.txt", "f[ab]o.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\*.txt", "foo*.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", NoEscape, false})
+		testcases = append(testcases, testcase{"f\\\\\\[ab\\\\\\]o.txt", "f\\[ab\\]o.txt", 0, true})
+	}
+
+	for _, tc := range testcases {
+		if m, err := Match(tc.pat, filepath.FromSlash(tc.name), tc.flags); m != tc.match {
+			if err != nil {
+				t.Error(err)
+			} else {
+				t.Errorf("Match(%q, %q, %d) != %v", tc.pat, tc.name, tc.flags, tc.match)
+			}
+		}
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	if _, err := Match("foo[bar", "...", 0); err == nil {
+		t.Error("Unexpected nil error")
+	}
+}

lib/ignore/cache.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package ignore
+
+import "time"
+
+type cache struct {
+	patterns []Pattern
+	entries  map[string]cacheEntry
+}
+
+type cacheEntry struct {
+	value  bool
+	access time.Time
+}
+
+func newCache(patterns []Pattern) *cache {
+	return &cache{
+		patterns: patterns,
+		entries:  make(map[string]cacheEntry),
+	}
+}
+
+func (c *cache) clean(d time.Duration) {
+	for k, v := range c.entries {
+		if time.Since(v.access) > d {
+			delete(c.entries, k)
+		}
+	}
+}
+
+func (c *cache) get(key string) (result, ok bool) {
+	res, ok := c.entries[key]
+	if ok {
+		res.access = time.Now()
+		c.entries[key] = res
+	}
+	return res.value, ok
+}
+
+func (c *cache) set(key string, val bool) {
+	c.entries[key] = cacheEntry{val, time.Now()}
+}
+
+func (c *cache) len() int {
+	l := len(c.entries)
+	return l
+}

lib/ignore/cache_test.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package ignore
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCache(t *testing.T) {
+	c := newCache(nil)
+
+	res, ok := c.get("nonexistent")
+	if res != false || ok != false {
+		t.Errorf("res %v, ok %v for nonexistent item", res, ok)
+	}
+
+	// Set and check some items
+
+	c.set("true", true)
+	c.set("false", false)
+
+	res, ok = c.get("true")
+	if res != true || ok != true {
+		t.Errorf("res %v, ok %v for true item", res, ok)
+	}
+
+	res, ok = c.get("false")
+	if res != false || ok != true {
+		t.Errorf("res %v, ok %v for false item", res, ok)
+	}
+
+	// Don't clean anything
+
+	c.clean(time.Second)
+
+	// Same values should exist
+
+	res, ok = c.get("true")
+	if res != true || ok != true {
+		t.Errorf("res %v, ok %v for true item", res, ok)
+	}
+
+	res, ok = c.get("false")
+	if res != false || ok != true {
+		t.Errorf("res %v, ok %v for false item", res, ok)
+	}
+
+	// Sleep and access, to get some data for clean
+
+	time.Sleep(500 * time.Millisecond)
+
+	c.get("true")
+
+	time.Sleep(100 * time.Millisecond)
+
+	// "false" was accessed ~600 ms ago, "true" was accessed ~100 ms ago.
+	// This should clean out "false" but not "true"
+
+	c.clean(300 * time.Millisecond)
+
+	// Same values should exist
+
+	_, ok = c.get("true")
+	if !ok {
+		t.Error("item should still exist")
+	}
+
+	_, ok = c.get("false")
+	if ok {
+		t.Errorf("item should have been cleaned")
+	}
+}

lib/ignore/ignore.go

@@ -0,0 +1,284 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package ignore
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/fnmatch"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type Pattern struct {
+	match   *regexp.Regexp
+	include bool
+}
+
+func (p Pattern) String() string {
+	if p.include {
+		return p.match.String()
+	}
+	return "(?exclude)" + p.match.String()
+}
+
+type Matcher struct {
+	patterns  []Pattern
+	withCache bool
+	matches   *cache
+	curHash   string
+	stop      chan struct{}
+	mut       sync.Mutex
+}
+
+func New(withCache bool) *Matcher {
+	m := &Matcher{
+		withCache: withCache,
+		stop:      make(chan struct{}),
+		mut:       sync.NewMutex(),
+	}
+	if withCache {
+		go m.clean(2 * time.Hour)
+	}
+	return m
+}
+
+func (m *Matcher) Load(file string) error {
+	// No locking, Parse() does the locking
+
+	fd, err := os.Open(file)
+	if err != nil {
+		// We do a parse with empty patterns to clear out the hash, cache etc.
+		m.Parse(&bytes.Buffer{}, file)
+		return err
+	}
+	defer fd.Close()
+
+	return m.Parse(fd, file)
+}
+
+func (m *Matcher) Parse(r io.Reader, file string) error {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	seen := map[string]bool{file: true}
+	patterns, err := parseIgnoreFile(r, file, seen)
+	// Error is saved and returned at the end. We process the patterns
+	// (possibly blank) anyway.
+
+	newHash := hashPatterns(patterns)
+	if newHash == m.curHash {
+		// We've already loaded exactly these patterns.
+		return err
+	}
+
+	m.curHash = newHash
+	m.patterns = patterns
+	if m.withCache {
+		m.matches = newCache(patterns)
+	}
+
+	return err
+}
+
+func (m *Matcher) Match(file string) (result bool) {
+	if m == nil {
+		return false
+	}
+
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	if len(m.patterns) == 0 {
+		return false
+	}
+
+	if m.matches != nil {
+		// Check the cache for a known result.
+		res, ok := m.matches.get(file)
+		if ok {
+			return res
+		}
+
+		// Update the cache with the result at return time
+		defer func() {
+			m.matches.set(file, result)
+		}()
+	}
+
+	// Check all the patterns for a match.
+	for _, pattern := range m.patterns {
+		if pattern.match.MatchString(file) {
+			return pattern.include
+		}
+	}
+
+	// Default to false.
+	return false
+}
+
+// Patterns return a list of the loaded regexp patterns, as strings
+func (m *Matcher) Patterns() []string {
+	if m == nil {
+		return nil
+	}
+
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	patterns := make([]string, len(m.patterns))
+	for i, pat := range m.patterns {
+		patterns[i] = pat.String()
+	}
+	return patterns
+}
+
+func (m *Matcher) Hash() string {
+	m.mut.Lock()
+	defer m.mut.Unlock()
+	return m.curHash
+}
+
+func (m *Matcher) Stop() {
+	close(m.stop)
+}
+
+func (m *Matcher) clean(d time.Duration) {
+	t := time.NewTimer(d / 2)
+	for {
+		select {
+		case <-m.stop:
+			return
+		case <-t.C:
+			m.mut.Lock()
+			if m.matches != nil {
+				m.matches.clean(d)
+			}
+			t.Reset(d / 2)
+			m.mut.Unlock()
+		}
+	}
+}
+
+func hashPatterns(patterns []Pattern) string {
+	h := md5.New()
+	for _, pat := range patterns {
+		h.Write([]byte(pat.String()))
+		h.Write([]byte("\n"))
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func loadIgnoreFile(file string, seen map[string]bool) ([]Pattern, error) {
+	if seen[file] {
+		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
+	}
+	seen[file] = true
+
+	fd, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	return parseIgnoreFile(fd, file, seen)
+}
+
+func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]Pattern, error) {
+	var patterns []Pattern
+
+	addPattern := func(line string) error {
+		include := true
+		if strings.HasPrefix(line, "!") {
+			line = line[1:]
+			include = false
+		}
+
+		if strings.HasPrefix(line, "/") {
+			// Pattern is rooted in the current dir only
+			exp, err := fnmatch.Convert(line[1:], fnmatch.PathName)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			patterns = append(patterns, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "**/") {
+			// Add the pattern as is, and without **/ so it matches in current dir
+			exp, err := fnmatch.Convert(line, fnmatch.PathName)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			patterns = append(patterns, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert(line[3:], fnmatch.PathName)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			patterns = append(patterns, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "#include ") {
+			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
+			includes, err := loadIgnoreFile(includeFile, seen)
+			if err != nil {
+				return err
+			}
+			patterns = append(patterns, includes...)
+		} else {
+			// Path name or pattern, add it so it matches files both in
+			// current directory and subdirs.
+			exp, err := fnmatch.Convert(line, fnmatch.PathName)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			patterns = append(patterns, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert("**/"+line, fnmatch.PathName)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			patterns = append(patterns, Pattern{exp, include})
+		}
+		return nil
+	}
+
+	scanner := bufio.NewScanner(fd)
+	var err error
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case line == "":
+			continue
+		case strings.HasPrefix(line, "//"):
+			continue
+		case strings.HasPrefix(line, "#"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/**"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/"):
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "**")
+			}
+		default:
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "/**")
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return patterns, nil
+}

lib/ignore/ignore_test.go

@@ -0,0 +1,511 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package ignore
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+func TestIgnore(t *testing.T) {
+	pats := New(true)
+	err := pats.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"afile", false},
+		{"bfile", true},
+		{"cfile", false},
+		{"dfile", false},
+		{"efile", true},
+		{"ffile", true},
+
+		{"dir1", false},
+		{filepath.Join("dir1", "cfile"), true},
+		{filepath.Join("dir1", "dfile"), false},
+		{filepath.Join("dir1", "efile"), true},
+		{filepath.Join("dir1", "ffile"), false},
+
+		{"dir2", false},
+		{filepath.Join("dir2", "cfile"), false},
+		{filepath.Join("dir2", "dfile"), true},
+		{filepath.Join("dir2", "efile"), true},
+		{filepath.Join("dir2", "ffile"), false},
+
+		{filepath.Join("dir3"), true},
+		{filepath.Join("dir3", "afile"), true},
+
+		{"lost+found", true},
+	}
+
+	for i, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
+		}
+	}
+}
+
+func TestExcludes(t *testing.T) {
+	stignore := `
+	!iex2
+	!ign1/ex
+	ign1
+	i*2
+	!ign2
+	`
+	pats := New(true)
+	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"ign1", true},
+		{"ign2", true},
+		{"ibla2", true},
+		{"iex2", false},
+		{filepath.Join("ign1", "ign"), true},
+		{filepath.Join("ign1", "ex"), false},
+		{filepath.Join("ign1", "iex2"), false},
+		{filepath.Join("iex2", "ign"), false},
+		{filepath.Join("foo", "bar", "ign1"), true},
+		{filepath.Join("foo", "bar", "ign2"), true},
+		{filepath.Join("foo", "bar", "iex2"), false},
+	}
+
+	for _, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect match for %s: %v != %v", tc.f, r, tc.r)
+		}
+	}
+}
+
+func TestBadPatterns(t *testing.T) {
+	var badPatterns = []string{
+		"[",
+		"/[",
+		"**/[",
+		"#include nonexistent",
+		"#include .stignore",
+		"!#include makesnosense",
+	}
+
+	for _, pat := range badPatterns {
+		err := New(true).Parse(bytes.NewBufferString(pat), ".stignore")
+		if err == nil {
+			t.Errorf("No error for pattern %q", pat)
+		}
+	}
+}
+
+func TestCaseSensitivity(t *testing.T) {
+	ign := New(true)
+	err := ign.Parse(bytes.NewBufferString("test"), ".stignore")
+	if err != nil {
+		t.Error(err)
+	}
+
+	match := []string{"test"}
+	dontMatch := []string{"foo"}
+
+	switch runtime.GOOS {
+	case "darwin", "windows":
+		match = append(match, "TEST", "Test", "tESt")
+	default:
+		dontMatch = append(dontMatch, "TEST", "Test", "tESt")
+	}
+
+	for _, tc := range match {
+		if !ign.Match(tc) {
+			t.Errorf("Incorrect match for %q: should be matched", tc)
+		}
+	}
+
+	for _, tc := range dontMatch {
+		if ign.Match(tc) {
+			t.Errorf("Incorrect match for %q: should not be matched", tc)
+		}
+	}
+}
+
+func TestCaching(t *testing.T) {
+	fd1, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd2, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer fd1.Close()
+	defer fd2.Close()
+	defer os.Remove(fd1.Name())
+	defer os.Remove(fd2.Name())
+
+	_, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd2.WriteString("/y/\n")
+
+	pats := New(true)
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pats.matches.len() != 0 {
+		t.Fatal("Expected empty cache")
+	}
+
+	if len(pats.patterns) != 4 {
+		t.Fatal("Incorrect number of patterns loaded", len(pats.patterns), "!=", 4)
+	}
+
+	// Cache some outcomes
+
+	for _, letter := range []string{"a", "b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	if pats.matches.len() != 4 {
+		t.Fatal("Expected 4 cached results")
+	}
+
+	// Reload file, expect old outcomes to be preserved
+
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pats.matches.len() != 4 {
+		t.Fatal("Expected 4 cached results")
+	}
+
+	// Modify the include file, expect empty cache
+
+	fd2.WriteString("/z/\n")
+
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pats.matches.len() != 0 {
+		t.Fatal("Expected 0 cached results")
+	}
+
+	// Cache some outcomes again
+
+	for _, letter := range []string{"b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	// Verify that outcomes preserved on next laod
+
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pats.matches.len() != 3 {
+		t.Fatal("Expected 3 cached results")
+	}
+
+	// Modify the root file, expect cache to be invalidated
+
+	fd1.WriteString("/a/\n")
+
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pats.matches.len() != 0 {
+		t.Fatal("Expected cache invalidation")
+	}
+
+	// Cache some outcomes again
+
+	for _, letter := range []string{"b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	// Verify that outcomes provided on next laod
+
+	err = pats.Load(fd1.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pats.matches.len() != 3 {
+		t.Fatal("Expected 3 cached results")
+	}
+}
+
+func TestCommentsAndBlankLines(t *testing.T) {
+	stignore := `
+	// foo
+	//bar
+
+	//!baz
+	//#dex
+
+	//                        ips
+
+
+	`
+	pats := New(true)
+	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(pats.patterns) > 0 {
+		t.Errorf("Expected no patterns")
+	}
+}
+
+var result bool
+
+func BenchmarkMatch(b *testing.B) {
+	stignore := `
+.frog
+.frog*
+.frogfox
+.whale
+.whale/*
+.dolphin
+.dolphin/*
+~ferret~.*
+.ferret.*
+flamingo.*
+flamingo
+*.crow
+*.crow
+	`
+	pats := New(false)
+	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		b.Error(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result = pats.Match("filename")
+	}
+}
+
+func BenchmarkMatchCached(b *testing.B) {
+	stignore := `
+.frog
+.frog*
+.frogfox
+.whale
+.whale/*
+.dolphin
+.dolphin/*
+~ferret~.*
+.ferret.*
+flamingo.*
+flamingo
+*.crow
+*.crow
+	`
+	// Caches per file, hence write the patterns to a file.
+	fd, err := ioutil.TempFile("", "")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	_, err = fd.WriteString(stignore)
+	defer fd.Close()
+	defer os.Remove(fd.Name())
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Load the patterns
+	pats := New(true)
+	err = pats.Load(fd.Name())
+	if err != nil {
+		b.Fatal(err)
+	}
+	// Cache the outcome for "filename"
+	pats.Match("filename")
+
+	// This load should now load the cached outcomes as the set of patterns
+	// has not changed.
+	err = pats.Load(fd.Name())
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result = pats.Match("filename")
+	}
+}
+
+func TestCacheReload(t *testing.T) {
+	fd, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer fd.Close()
+	defer os.Remove(fd.Name())
+
+	// Ignore file matches f1 and f2
+
+	_, err = fd.WriteString("f1\nf2\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pats := New(true)
+	err = pats.Load(fd.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that both are ignored
+
+	if !pats.Match("f1") {
+		t.Error("Unexpected non-match for f1")
+	}
+	if !pats.Match("f2") {
+		t.Error("Unexpected non-match for f2")
+	}
+	if pats.Match("f3") {
+		t.Error("Unexpected match for f3")
+	}
+
+	// Rewrite file to match f1 and f3
+
+	err = fd.Truncate(0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.Seek(0, os.SEEK_SET)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fd.WriteString("f1\nf3\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = pats.Load(fd.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the new patterns are in effect
+
+	if !pats.Match("f1") {
+		t.Error("Unexpected non-match for f1")
+	}
+	if pats.Match("f2") {
+		t.Error("Unexpected match for f2")
+	}
+	if !pats.Match("f3") {
+		t.Error("Unexpected non-match for f3")
+	}
+}
+
+func TestHash(t *testing.T) {
+	p1 := New(true)
+	err := p1.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Same list of patterns as testdata/.stignore, after expansion
+	stignore := `
+	dir2/dfile
+	dir3
+	bfile
+	dir1/cfile
+	**/efile
+	/ffile
+	lost+found
+	`
+	p2 := New(true)
+	err = p2.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Not same list of patterns
+	stignore = `
+	dir2/dfile
+	dir3
+	bfile
+	dir1/cfile
+	/ffile
+	lost+found
+	`
+	p3 := New(true)
+	err = p3.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if p1.Hash() == "" {
+		t.Error("p1 hash blank")
+	}
+	if p2.Hash() == "" {
+		t.Error("p2 hash blank")
+	}
+	if p3.Hash() == "" {
+		t.Error("p3 hash blank")
+	}
+	if p1.Hash() != p2.Hash() {
+		t.Error("p1-p2 hashes differ")
+	}
+	if p1.Hash() == p3.Hash() {
+		t.Error("p1-p3 hashes same")
+	}
+}
+
+func TestHashOfEmpty(t *testing.T) {
+	p1 := New(true)
+	err := p1.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	firstHash := p1.Hash()
+
+	// Reloading with a non-existent file should empty the patterns and
+	// recalculate the hash. d41d8cd98f00b204e9800998ecf8427e is the md5 of
+	// nothing.
+
+	p1.Load("file/does/not/exist")
+	secondHash := p1.Hash()
+
+	if firstHash == secondHash {
+		t.Error("hash did not change")
+	}
+	if secondHash != "d41d8cd98f00b204e9800998ecf8427e" {
+		t.Error("second hash is not hash of empty string")
+	}
+	if len(p1.patterns) != 0 {
+		t.Error("there are more than zero patterns")
+	}
+}

lib/ignore/testdata/.stignore

@@ -0,0 +1,7 @@
+#include excludes
+
+bfile
+dir1/cfile
+**/efile
+/ffile
+lost+found

lib/ignore/testdata/dir3/cfile

@@ -0,0 +1 @@
+baz

lib/ignore/testdata/dir3/dfile

@@ -0,0 +1 @@
+quux

lib/ignore/testdata/excludes

@@ -0,0 +1,2 @@
+dir2/dfile
+#include further-excludes

lib/ignore/testdata/further-excludes

@@ -0,0 +1 @@
+dir3

lib/model/.gitignore

@@ -0,0 +1,2 @@
+.stfolder
+.stignore

lib/model/debug.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

lib/model/deviceactivity.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+// deviceActivity tracks the number of outstanding requests per device and can
+// answer which device is least busy. It is safe for use from multiple
+// goroutines.
+type deviceActivity struct {
+	act map[protocol.DeviceID]int
+	mut sync.Mutex
+}
+
+func newDeviceActivity() *deviceActivity {
+	return &deviceActivity{
+		act: make(map[protocol.DeviceID]int),
+		mut: sync.NewMutex(),
+	}
+}
+
+func (m *deviceActivity) leastBusy(availability []protocol.DeviceID) protocol.DeviceID {
+	m.mut.Lock()
+	low := 2<<30 - 1
+	var selected protocol.DeviceID
+	for _, device := range availability {
+		if usage := m.act[device]; usage < low {
+			low = usage
+			selected = device
+		}
+	}
+	m.mut.Unlock()
+	return selected
+}
+
+func (m *deviceActivity) using(device protocol.DeviceID) {
+	m.mut.Lock()
+	m.act[device]++
+	m.mut.Unlock()
+}
+
+func (m *deviceActivity) done(device protocol.DeviceID) {
+	m.mut.Lock()
+	m.act[device]--
+	m.mut.Unlock()
+}

lib/model/deviceactivity_test.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"testing"
+
+	"github.com/syncthing/protocol"
+)
+
+func TestDeviceActivity(t *testing.T) {
+	n0 := protocol.DeviceID([32]byte{1, 2, 3, 4})
+	n1 := protocol.DeviceID([32]byte{5, 6, 7, 8})
+	n2 := protocol.DeviceID([32]byte{9, 10, 11, 12})
+	devices := []protocol.DeviceID{n0, n1, n2}
+	na := newDeviceActivity()
+
+	if lb := na.leastBusy(devices); lb != n0 {
+		t.Errorf("Least busy device should be n0 (%v) not %v", n0, lb)
+	}
+	if lb := na.leastBusy(devices); lb != n0 {
+		t.Errorf("Least busy device should still be n0 (%v) not %v", n0, lb)
+	}
+
+	na.using(na.leastBusy(devices))
+	if lb := na.leastBusy(devices); lb != n1 {
+		t.Errorf("Least busy device should be n1 (%v) not %v", n1, lb)
+	}
+
+	na.using(na.leastBusy(devices))
+	if lb := na.leastBusy(devices); lb != n2 {
+		t.Errorf("Least busy device should be n2 (%v) not %v", n2, lb)
+	}
+
+	na.using(na.leastBusy(devices))
+	if lb := na.leastBusy(devices); lb != n0 {
+		t.Errorf("Least busy device should be n0 (%v) not %v", n0, lb)
+	}
+
+	na.done(n1)
+	if lb := na.leastBusy(devices); lb != n1 {
+		t.Errorf("Least busy device should be n1 (%v) not %v", n1, lb)
+	}
+
+	na.done(n2)
+	if lb := na.leastBusy(devices); lb != n1 {
+		t.Errorf("Least busy device should still be n1 (%v) not %v", n1, lb)
+	}
+
+	na.done(n0)
+	if lb := na.leastBusy(devices); lb != n0 {
+		t.Errorf("Least busy device should be n0 (%v) not %v", n0, lb)
+	}
+}

lib/model/doc.go

@@ -0,0 +1,8 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package model implements folder abstraction and file pulling mechanisms
+package model

lib/model/folderstate.go

@@ -0,0 +1,135 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"time"
+
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type folderState int
+
+const (
+	FolderIdle folderState = iota
+	FolderScanning
+	FolderSyncing
+	FolderError
+)
+
+func (s folderState) String() string {
+	switch s {
+	case FolderIdle:
+		return "idle"
+	case FolderScanning:
+		return "scanning"
+	case FolderSyncing:
+		return "syncing"
+	case FolderError:
+		return "error"
+	default:
+		return "unknown"
+	}
+}
+
+type stateTracker struct {
+	folder string
+
+	mut     sync.Mutex
+	current folderState
+	err     error
+	changed time.Time
+}
+
+// setState sets the new folder state, for states other than FolderError.
+func (s *stateTracker) setState(newState folderState) {
+	if newState == FolderError {
+		panic("must use setError")
+	}
+
+	s.mut.Lock()
+	if newState != s.current {
+		/* This should hold later...
+		if s.current != FolderIdle && (newState == FolderScanning || newState == FolderSyncing) {
+			panic("illegal state transition " + s.current.String() + " -> " + newState.String())
+		}
+		*/
+
+		eventData := map[string]interface{}{
+			"folder": s.folder,
+			"to":     newState.String(),
+			"from":   s.current.String(),
+		}
+
+		if !s.changed.IsZero() {
+			eventData["duration"] = time.Since(s.changed).Seconds()
+		}
+
+		s.current = newState
+		s.changed = time.Now()
+
+		events.Default.Log(events.StateChanged, eventData)
+	}
+	s.mut.Unlock()
+}
+
+// getState returns the current state, the time when it last changed, and the
+// current error or nil.
+func (s *stateTracker) getState() (current folderState, changed time.Time, err error) {
+	s.mut.Lock()
+	current, changed, err = s.current, s.changed, s.err
+	s.mut.Unlock()
+	return
+}
+
+// setError sets the folder state to FolderError with the specified error.
+func (s *stateTracker) setError(err error) {
+	s.mut.Lock()
+	if s.current != FolderError || s.err.Error() != err.Error() {
+		eventData := map[string]interface{}{
+			"folder": s.folder,
+			"to":     FolderError.String(),
+			"from":   s.current.String(),
+			"error":  err.Error(),
+		}
+
+		if !s.changed.IsZero() {
+			eventData["duration"] = time.Since(s.changed).Seconds()
+		}
+
+		s.current = FolderError
+		s.err = err
+		s.changed = time.Now()
+
+		events.Default.Log(events.StateChanged, eventData)
+	}
+	s.mut.Unlock()
+}
+
+// clearError sets the folder state to FolderIdle and clears the error
+func (s *stateTracker) clearError() {
+	s.mut.Lock()
+	if s.current == FolderError {
+		eventData := map[string]interface{}{
+			"folder": s.folder,
+			"to":     FolderIdle.String(),
+			"from":   s.current.String(),
+		}
+
+		if !s.changed.IsZero() {
+			eventData["duration"] = time.Since(s.changed).Seconds()
+		}
+
+		s.current = FolderIdle
+		s.err = nil
+		s.changed = time.Now()
+
+		events.Default.Log(events.StateChanged, eventData)
+	}
+	s.mut.Unlock()
+}

lib/model/progressemitter.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type ProgressEmitter struct {
+	registry map[string]*sharedPullerState
+	interval time.Duration
+	last     map[string]map[string]*pullerProgress
+	mut      sync.Mutex
+
+	timer *time.Timer
+
+	stop chan struct{}
+}
+
+// NewProgressEmitter creates a new progress emitter which emits
+// DownloadProgress events every interval.
+func NewProgressEmitter(cfg *config.Wrapper) *ProgressEmitter {
+	t := &ProgressEmitter{
+		stop:     make(chan struct{}),
+		registry: make(map[string]*sharedPullerState),
+		last:     make(map[string]map[string]*pullerProgress),
+		timer:    time.NewTimer(time.Millisecond),
+		mut:      sync.NewMutex(),
+	}
+
+	t.CommitConfiguration(config.Configuration{}, cfg.Raw())
+	cfg.Subscribe(t)
+
+	return t
+}
+
+// Serve starts the progress emitter which starts emitting DownloadProgress
+// events as the progress happens.
+func (t *ProgressEmitter) Serve() {
+	for {
+		select {
+		case <-t.stop:
+			if debug {
+				l.Debugln("progress emitter: stopping")
+			}
+			return
+		case <-t.timer.C:
+			t.mut.Lock()
+			if debug {
+				l.Debugln("progress emitter: timer - looking after", len(t.registry))
+			}
+			output := make(map[string]map[string]*pullerProgress)
+			for _, puller := range t.registry {
+				if output[puller.folder] == nil {
+					output[puller.folder] = make(map[string]*pullerProgress)
+				}
+				output[puller.folder][puller.file.Name] = puller.Progress()
+			}
+			if !reflect.DeepEqual(t.last, output) {
+				events.Default.Log(events.DownloadProgress, output)
+				t.last = output
+				if debug {
+					l.Debugf("progress emitter: emitting %#v", output)
+				}
+			} else if debug {
+				l.Debugln("progress emitter: nothing new")
+			}
+			if len(t.registry) != 0 {
+				t.timer.Reset(t.interval)
+			}
+			t.mut.Unlock()
+		}
+	}
+}
+
+// VerifyConfiguration implements the config.Committer interface
+func (t *ProgressEmitter) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+// CommitConfiguration implements the config.Committer interface
+func (t *ProgressEmitter) CommitConfiguration(from, to config.Configuration) bool {
+	t.mut.Lock()
+	defer t.mut.Unlock()
+
+	t.interval = time.Duration(to.Options.ProgressUpdateIntervalS) * time.Second
+	if debug {
+		l.Debugln("progress emitter: updated interval", t.interval)
+	}
+
+	return true
+}
+
+// Stop stops the emitter.
+func (t *ProgressEmitter) Stop() {
+	t.stop <- struct{}{}
+}
+
+// Register a puller with the emitter which will start broadcasting pullers
+// progress.
+func (t *ProgressEmitter) Register(s *sharedPullerState) {
+	t.mut.Lock()
+	defer t.mut.Unlock()
+	if debug {
+		l.Debugln("progress emitter: registering", s.folder, s.file.Name)
+	}
+	if len(t.registry) == 0 {
+		t.timer.Reset(t.interval)
+	}
+	t.registry[filepath.Join(s.folder, s.file.Name)] = s
+}
+
+// Deregister a puller which will stop broadcasting pullers state.
+func (t *ProgressEmitter) Deregister(s *sharedPullerState) {
+	t.mut.Lock()
+	defer t.mut.Unlock()
+	if debug {
+		l.Debugln("progress emitter: deregistering", s.folder, s.file.Name)
+	}
+	delete(t.registry, filepath.Join(s.folder, s.file.Name))
+}
+
+// BytesCompleted returns the number of bytes completed in the given folder.
+func (t *ProgressEmitter) BytesCompleted(folder string) (bytes int64) {
+	t.mut.Lock()
+	defer t.mut.Unlock()
+
+	for _, s := range t.registry {
+		if s.folder == folder {
+			bytes += s.Progress().BytesDone
+		}
+	}
+	if debug {
+		l.Debugf("progress emitter: bytes completed for %s: %d", folder, bytes)
+	}
+	return
+}
+
+func (t *ProgressEmitter) String() string {
+	return fmt.Sprintf("ProgressEmitter@%p", t)
+}

lib/model/progressemitter_test.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+var timeout = 10 * time.Millisecond
+
+func expectEvent(w *events.Subscription, t *testing.T, size int) {
+	event, err := w.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if event.Type != events.DownloadProgress {
+		t.Fatal("Unexpected event:", event)
+	}
+	data := event.Data.(map[string]map[string]*pullerProgress)
+	if len(data) != size {
+		t.Fatal("Unexpected event data size:", data)
+	}
+}
+
+func expectTimeout(w *events.Subscription, t *testing.T) {
+	_, err := w.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestProgressEmitter(t *testing.T) {
+	w := events.Default.Subscribe(events.DownloadProgress)
+
+	c := config.Wrap("/tmp/test", config.Configuration{})
+	c.SetOptions(config.OptionsConfiguration{
+		ProgressUpdateIntervalS: 0,
+	})
+
+	p := NewProgressEmitter(c)
+	go p.Serve()
+
+	expectTimeout(w, t)
+
+	s := sharedPullerState{
+		mut: sync.NewMutex(),
+	}
+	p.Register(&s)
+
+	expectEvent(w, t, 1)
+	expectTimeout(w, t)
+
+	s.copyDone()
+
+	expectEvent(w, t, 1)
+	expectTimeout(w, t)
+
+	s.copiedFromOrigin()
+
+	expectEvent(w, t, 1)
+	expectTimeout(w, t)
+
+	s.pullStarted()
+
+	expectEvent(w, t, 1)
+	expectTimeout(w, t)
+
+	s.pullDone()
+
+	expectEvent(w, t, 1)
+	expectTimeout(w, t)
+
+	p.Deregister(&s)
+
+	expectEvent(w, t, 0)
+	expectTimeout(w, t)
+
+}

lib/model/queue.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"math/rand"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type jobQueue struct {
+	progress []string
+	queued   []jobQueueEntry
+	mut      sync.Mutex
+}
+
+type jobQueueEntry struct {
+	name     string
+	size     int64
+	modified int64
+}
+
+func newJobQueue() *jobQueue {
+	return &jobQueue{
+		mut: sync.NewMutex(),
+	}
+}
+
+func (q *jobQueue) Push(file string, size, modified int64) {
+	q.mut.Lock()
+	q.queued = append(q.queued, jobQueueEntry{file, size, modified})
+	q.mut.Unlock()
+}
+
+func (q *jobQueue) Pop() (string, bool) {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	if len(q.queued) == 0 {
+		return "", false
+	}
+
+	f := q.queued[0].name
+	q.queued = q.queued[1:]
+	q.progress = append(q.progress, f)
+
+	return f, true
+}
+
+func (q *jobQueue) BringToFront(filename string) {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	for i, cur := range q.queued {
+		if cur.name == filename {
+			if i > 0 {
+				// Shift the elements before the selected element one step to
+				// the right, overwriting the selected element
+				copy(q.queued[1:i+1], q.queued[0:])
+				// Put the selected element at the front
+				q.queued[0] = cur
+			}
+			return
+		}
+	}
+}
+
+func (q *jobQueue) Done(file string) {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	for i := range q.progress {
+		if q.progress[i] == file {
+			copy(q.progress[i:], q.progress[i+1:])
+			q.progress = q.progress[:len(q.progress)-1]
+			return
+		}
+	}
+}
+
+func (q *jobQueue) Jobs() ([]string, []string) {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	progress := make([]string, len(q.progress))
+	copy(progress, q.progress)
+
+	queued := make([]string, len(q.queued))
+	for i := range q.queued {
+		queued[i] = q.queued[i].name
+	}
+
+	return progress, queued
+}
+
+func (q *jobQueue) Shuffle() {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	l := len(q.queued)
+	for i := range q.queued {
+		r := rand.Intn(l)
+		q.queued[i], q.queued[r] = q.queued[r], q.queued[i]
+	}
+}
+
+func (q *jobQueue) SortSmallestFirst() {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	sort.Sort(smallestFirst(q.queued))
+}
+
+func (q *jobQueue) SortLargestFirst() {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	sort.Sort(sort.Reverse(smallestFirst(q.queued)))
+}
+
+func (q *jobQueue) SortOldestFirst() {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	sort.Sort(oldestFirst(q.queued))
+}
+
+func (q *jobQueue) SortNewestFirst() {
+	q.mut.Lock()
+	defer q.mut.Unlock()
+
+	sort.Sort(sort.Reverse(oldestFirst(q.queued)))
+}
+
+// The usual sort.Interface boilerplate
+
+type smallestFirst []jobQueueEntry
+
+func (q smallestFirst) Len() int           { return len(q) }
+func (q smallestFirst) Less(a, b int) bool { return q[a].size < q[b].size }
+func (q smallestFirst) Swap(a, b int)      { q[a], q[b] = q[b], q[a] }
+
+type oldestFirst []jobQueueEntry
+
+func (q oldestFirst) Len() int           { return len(q) }
+func (q oldestFirst) Less(a, b int) bool { return q[a].modified < q[b].modified }
+func (q oldestFirst) Swap(a, b int)      { q[a], q[b] = q[b], q[a] }

lib/model/queue_test.go

@@ -0,0 +1,280 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestJobQueue(t *testing.T) {
+	// Some random actions
+	q := newJobQueue()
+	q.Push("f1", 0, 0)
+	q.Push("f2", 0, 0)
+	q.Push("f3", 0, 0)
+	q.Push("f4", 0, 0)
+
+	progress, queued := q.Jobs()
+	if len(progress) != 0 || len(queued) != 4 {
+		t.Fatal("Wrong length")
+	}
+
+	for i := 1; i < 5; i++ {
+		n, ok := q.Pop()
+		if !ok || n != fmt.Sprintf("f%d", i) {
+			t.Fatal("Wrong element")
+		}
+		progress, queued = q.Jobs()
+		if len(progress) != 1 || len(queued) != 3 {
+			t.Log(progress)
+			t.Log(queued)
+			t.Fatal("Wrong length")
+		}
+
+		q.Done(n)
+		progress, queued = q.Jobs()
+		if len(progress) != 0 || len(queued) != 3 {
+			t.Fatal("Wrong length", len(progress), len(queued))
+		}
+
+		q.Push(n, 0, 0)
+		progress, queued = q.Jobs()
+		if len(progress) != 0 || len(queued) != 4 {
+			t.Fatal("Wrong length")
+		}
+
+		q.Done("f5") // Does not exist
+		progress, queued = q.Jobs()
+		if len(progress) != 0 || len(queued) != 4 {
+			t.Fatal("Wrong length")
+		}
+	}
+
+	if len(q.progress) > 0 || len(q.queued) != 4 {
+		t.Fatal("Wrong length")
+	}
+
+	for i := 4; i > 0; i-- {
+		progress, queued = q.Jobs()
+		if len(progress) != 4-i || len(queued) != i {
+			t.Fatal("Wrong length")
+		}
+
+		s := fmt.Sprintf("f%d", i)
+
+		q.BringToFront(s)
+		progress, queued = q.Jobs()
+		if len(progress) != 4-i || len(queued) != i {
+			t.Fatal("Wrong length")
+		}
+
+		n, ok := q.Pop()
+		if !ok || n != s {
+			t.Fatal("Wrong element")
+		}
+		progress, queued = q.Jobs()
+		if len(progress) != 5-i || len(queued) != i-1 {
+			t.Fatal("Wrong length")
+		}
+
+		q.Done("f5") // Does not exist
+		progress, queued = q.Jobs()
+		if len(progress) != 5-i || len(queued) != i-1 {
+			t.Fatal("Wrong length")
+		}
+	}
+
+	_, ok := q.Pop()
+	if len(q.progress) != 4 || ok {
+		t.Fatal("Wrong length")
+	}
+
+	q.Done("f1")
+	q.Done("f2")
+	q.Done("f3")
+	q.Done("f4")
+	q.Done("f5") // Does not exist
+
+	_, ok = q.Pop()
+	if len(q.progress) != 0 || ok {
+		t.Fatal("Wrong length")
+	}
+
+	progress, queued = q.Jobs()
+	if len(progress) != 0 || len(queued) != 0 {
+		t.Fatal("Wrong length")
+	}
+	q.BringToFront("")
+	q.Done("f5") // Does not exist
+	progress, queued = q.Jobs()
+	if len(progress) != 0 || len(queued) != 0 {
+		t.Fatal("Wrong length")
+	}
+}
+
+func TestBringToFront(t *testing.T) {
+	q := newJobQueue()
+	q.Push("f1", 0, 0)
+	q.Push("f2", 0, 0)
+	q.Push("f3", 0, 0)
+	q.Push("f4", 0, 0)
+
+	_, queued := q.Jobs()
+	if !reflect.DeepEqual(queued, []string{"f1", "f2", "f3", "f4"}) {
+		t.Errorf("Incorrect order %v at start", queued)
+	}
+
+	q.BringToFront("f1") // corner case: does nothing
+
+	_, queued = q.Jobs()
+	if !reflect.DeepEqual(queued, []string{"f1", "f2", "f3", "f4"}) {
+		t.Errorf("Incorrect order %v", queued)
+	}
+
+	q.BringToFront("f3")
+
+	_, queued = q.Jobs()
+	if !reflect.DeepEqual(queued, []string{"f3", "f1", "f2", "f4"}) {
+		t.Errorf("Incorrect order %v", queued)
+	}
+
+	q.BringToFront("f2")
+
+	_, queued = q.Jobs()
+	if !reflect.DeepEqual(queued, []string{"f2", "f3", "f1", "f4"}) {
+		t.Errorf("Incorrect order %v", queued)
+	}
+
+	q.BringToFront("f4") // corner case: last element
+
+	_, queued = q.Jobs()
+	if !reflect.DeepEqual(queued, []string{"f4", "f2", "f3", "f1"}) {
+		t.Errorf("Incorrect order %v", queued)
+	}
+}
+
+func TestShuffle(t *testing.T) {
+	q := newJobQueue()
+	q.Push("f1", 0, 0)
+	q.Push("f2", 0, 0)
+	q.Push("f3", 0, 0)
+	q.Push("f4", 0, 0)
+
+	// This test will fail once in eight million times (1 / (4!)^5) :)
+	for i := 0; i < 5; i++ {
+		q.Shuffle()
+		_, queued := q.Jobs()
+		if l := len(queued); l != 4 {
+			t.Fatalf("Weird length %d returned from Jobs()", l)
+		}
+
+		t.Logf("%v", queued)
+		if !reflect.DeepEqual(queued, []string{"f1", "f2", "f3", "f4"}) {
+			// The queue was shuffled
+			return
+		}
+	}
+
+	t.Error("Queue was not shuffled after five attempts.")
+}
+
+func TestSortBySize(t *testing.T) {
+	q := newJobQueue()
+	q.Push("f1", 20, 0)
+	q.Push("f2", 40, 0)
+	q.Push("f3", 30, 0)
+	q.Push("f4", 10, 0)
+
+	q.SortSmallestFirst()
+
+	_, actual := q.Jobs()
+	if l := len(actual); l != 4 {
+		t.Fatalf("Weird length %d returned from Jobs()", l)
+	}
+	expected := []string{"f4", "f1", "f3", "f2"}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("SortSmallestFirst(): %#v != %#v", actual, expected)
+	}
+
+	q.SortLargestFirst()
+
+	_, actual = q.Jobs()
+	if l := len(actual); l != 4 {
+		t.Fatalf("Weird length %d returned from Jobs()", l)
+	}
+	expected = []string{"f2", "f3", "f1", "f4"}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("SortLargestFirst(): %#v != %#v", actual, expected)
+	}
+}
+
+func TestSortByAge(t *testing.T) {
+	q := newJobQueue()
+	q.Push("f1", 0, 20)
+	q.Push("f2", 0, 40)
+	q.Push("f3", 0, 30)
+	q.Push("f4", 0, 10)
+
+	q.SortOldestFirst()
+
+	_, actual := q.Jobs()
+	if l := len(actual); l != 4 {
+		t.Fatalf("Weird length %d returned from Jobs()", l)
+	}
+	expected := []string{"f4", "f1", "f3", "f2"}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("SortOldestFirst(): %#v != %#v", actual, expected)
+	}
+
+	q.SortNewestFirst()
+
+	_, actual = q.Jobs()
+	if l := len(actual); l != 4 {
+		t.Fatalf("Weird length %d returned from Jobs()", l)
+	}
+	expected = []string{"f2", "f3", "f1", "f4"}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("SortNewestFirst(): %#v != %#v", actual, expected)
+	}
+}
+
+func BenchmarkJobQueueBump(b *testing.B) {
+	files := genFiles(b.N)
+
+	q := newJobQueue()
+	for _, f := range files {
+		q.Push(f.Name, 0, 0)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		q.BringToFront(files[i].Name)
+	}
+}
+
+func BenchmarkJobQueuePushPopDone10k(b *testing.B) {
+	files := genFiles(10000)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		q := newJobQueue()
+		for _, f := range files {
+			q.Push(f.Name, 0, 0)
+		}
+		for _ = range files {
+			n, _ := q.Pop()
+			q.Done(n)
+		}
+	}
+
+}

lib/model/rofolder.go

@@ -0,0 +1,164 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type roFolder struct {
+	stateTracker
+
+	folder    string
+	intv      time.Duration
+	timer     *time.Timer
+	model     *Model
+	stop      chan struct{}
+	scanNow   chan rescanRequest
+	delayScan chan time.Duration
+}
+
+type rescanRequest struct {
+	subs []string
+	err  chan error
+}
+
+func newROFolder(model *Model, folder string, interval time.Duration) *roFolder {
+	return &roFolder{
+		stateTracker: stateTracker{
+			folder: folder,
+			mut:    sync.NewMutex(),
+		},
+		folder:    folder,
+		intv:      interval,
+		timer:     time.NewTimer(time.Millisecond),
+		model:     model,
+		stop:      make(chan struct{}),
+		scanNow:   make(chan rescanRequest),
+		delayScan: make(chan time.Duration),
+	}
+}
+
+func (s *roFolder) Serve() {
+	if debug {
+		l.Debugln(s, "starting")
+		defer l.Debugln(s, "exiting")
+	}
+
+	defer func() {
+		s.timer.Stop()
+	}()
+
+	reschedule := func() {
+		if s.intv == 0 {
+			return
+		}
+		// Sleep a random time between 3/4 and 5/4 of the configured interval.
+		sleepNanos := (s.intv.Nanoseconds()*3 + rand.Int63n(2*s.intv.Nanoseconds())) / 4
+		s.timer.Reset(time.Duration(sleepNanos) * time.Nanosecond)
+	}
+
+	initialScanCompleted := false
+	for {
+		select {
+		case <-s.stop:
+			return
+
+		case <-s.timer.C:
+			if err := s.model.CheckFolderHealth(s.folder); err != nil {
+				l.Infoln("Skipping folder", s.folder, "scan due to folder error:", err)
+				reschedule()
+				continue
+			}
+
+			if debug {
+				l.Debugln(s, "rescan")
+			}
+
+			if err := s.model.internalScanFolderSubs(s.folder, nil); err != nil {
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by setError.
+				s.setError(err)
+				reschedule()
+				continue
+			}
+
+			if !initialScanCompleted {
+				l.Infoln("Completed initial scan (ro) of folder", s.folder)
+				initialScanCompleted = true
+			}
+
+			if s.intv == 0 {
+				continue
+			}
+
+			reschedule()
+
+		case req := <-s.scanNow:
+			if err := s.model.CheckFolderHealth(s.folder); err != nil {
+				l.Infoln("Skipping folder", s.folder, "scan due to folder error:", err)
+				req.err <- err
+				continue
+			}
+
+			if debug {
+				l.Debugln(s, "forced rescan")
+			}
+
+			if err := s.model.internalScanFolderSubs(s.folder, req.subs); err != nil {
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by setError.
+				s.setError(err)
+				req.err <- err
+				continue
+			}
+
+			req.err <- nil
+
+		case next := <-s.delayScan:
+			s.timer.Reset(next)
+		}
+	}
+}
+
+func (s *roFolder) Stop() {
+	close(s.stop)
+}
+
+func (s *roFolder) IndexUpdated() {
+}
+
+func (s *roFolder) Scan(subs []string) error {
+	req := rescanRequest{
+		subs: subs,
+		err:  make(chan error),
+	}
+	s.scanNow <- req
+	return <-req.err
+}
+
+func (s *roFolder) String() string {
+	return fmt.Sprintf("roFolder/%s@%p", s.folder, s)
+}
+
+func (s *roFolder) BringToFront(string) {}
+
+func (s *roFolder) Jobs() ([]string, []string) {
+	return nil, nil
+}
+
+func (s *roFolder) DelayScan(next time.Duration) {
+	s.delayScan <- next
+}

lib/model/rwfolder_test.go

@@ -0,0 +1,546 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/scanner"
+	"github.com/syncthing/syncthing/lib/sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func init() {
+	// We do this to make sure that the temp file required for the tests does
+	// not get removed during the tests.
+	future := time.Now().Add(time.Hour)
+	err := os.Chtimes(filepath.Join("testdata", defTempNamer.TempName("file")), future, future)
+	if err != nil {
+		panic(err)
+	}
+}
+
+var blocks = []protocol.BlockInfo{
+	{Hash: []uint8{0xfa, 0x43, 0x23, 0x9b, 0xce, 0xe7, 0xb9, 0x7c, 0xa6, 0x2f, 0x0, 0x7c, 0xc6, 0x84, 0x87, 0x56, 0xa, 0x39, 0xe1, 0x9f, 0x74, 0xf3, 0xdd, 0xe7, 0x48, 0x6d, 0xb3, 0xf9, 0x8d, 0xf8, 0xe4, 0x71}}, // Zero'ed out block
+	{Offset: 0, Size: 0x20000, Hash: []uint8{0x7e, 0xad, 0xbc, 0x36, 0xae, 0xbb, 0xcf, 0x74, 0x43, 0xe2, 0x7a, 0x5a, 0x4b, 0xb8, 0x5b, 0xce, 0xe6, 0x9e, 0x1e, 0x10, 0xf9, 0x8a, 0xbc, 0x77, 0x95, 0x2, 0x29, 0x60, 0x9e, 0x96, 0xae, 0x6c}},
+	{Offset: 131072, Size: 0x20000, Hash: []uint8{0x3c, 0xc4, 0x20, 0xf4, 0xb, 0x2e, 0xcb, 0xb9, 0x5d, 0xce, 0x34, 0xa8, 0xc3, 0x92, 0xea, 0xf3, 0xda, 0x88, 0x33, 0xee, 0x7a, 0xb6, 0xe, 0xf1, 0x82, 0x5e, 0xb0, 0xa9, 0x26, 0xa9, 0xc0, 0xef}},
+	{Offset: 262144, Size: 0x20000, Hash: []uint8{0x76, 0xa8, 0xc, 0x69, 0xd7, 0x5c, 0x52, 0xfd, 0xdf, 0x55, 0xef, 0x44, 0xc1, 0xd6, 0x25, 0x48, 0x4d, 0x98, 0x48, 0x4d, 0xaa, 0x50, 0xf6, 0x6b, 0x32, 0x47, 0x55, 0x81, 0x6b, 0xed, 0xee, 0xfb}},
+	{Offset: 393216, Size: 0x20000, Hash: []uint8{0x44, 0x1e, 0xa4, 0xf2, 0x8d, 0x1f, 0xc3, 0x1b, 0x9d, 0xa5, 0x18, 0x5e, 0x59, 0x1b, 0xd8, 0x5c, 0xba, 0x7d, 0xb9, 0x8d, 0x70, 0x11, 0x5c, 0xea, 0xa1, 0x57, 0x4d, 0xcb, 0x3c, 0x5b, 0xf8, 0x6c}},
+	{Offset: 524288, Size: 0x20000, Hash: []uint8{0x8, 0x40, 0xd0, 0x5e, 0x80, 0x0, 0x0, 0x7c, 0x8b, 0xb3, 0x8b, 0xf7, 0x7b, 0x23, 0x26, 0x28, 0xab, 0xda, 0xcf, 0x86, 0x8f, 0xc2, 0x8a, 0x39, 0xc6, 0xe6, 0x69, 0x59, 0x97, 0xb6, 0x1a, 0x43}},
+	{Offset: 655360, Size: 0x20000, Hash: []uint8{0x38, 0x8e, 0x44, 0xcb, 0x30, 0xd8, 0x90, 0xf, 0xce, 0x7, 0x4b, 0x58, 0x86, 0xde, 0xce, 0x59, 0xa2, 0x46, 0xd2, 0xf9, 0xba, 0xaf, 0x35, 0x87, 0x38, 0xdf, 0xd2, 0xd, 0xf9, 0x45, 0xed, 0x91}},
+	{Offset: 786432, Size: 0x20000, Hash: []uint8{0x32, 0x28, 0xcd, 0xf, 0x37, 0x21, 0xe5, 0xd4, 0x1e, 0x58, 0x87, 0x73, 0x8e, 0x36, 0xdf, 0xb2, 0x70, 0x78, 0x56, 0xc3, 0x42, 0xff, 0xf7, 0x8f, 0x37, 0x95, 0x0, 0x26, 0xa, 0xac, 0x54, 0x72}},
+	{Offset: 917504, Size: 0x20000, Hash: []uint8{0x96, 0x6b, 0x15, 0x6b, 0xc4, 0xf, 0x19, 0x18, 0xca, 0xbb, 0x5f, 0xd6, 0xbb, 0xa2, 0xc6, 0x2a, 0xac, 0xbb, 0x8a, 0xb9, 0xce, 0xec, 0x4c, 0xdb, 0x78, 0xec, 0x57, 0x5d, 0x33, 0xf9, 0x8e, 0xaf}},
+}
+
+// Layout of the files: (indexes from the above array)
+// 12345678 - Required file
+// 02005008 - Existing file (currently in the index)
+// 02340070 - Temp file on the disk
+
+func TestHandleFile(t *testing.T) {
+	// After the diff between required and existing we should:
+	// Copy: 2, 5, 8
+	// Pull: 1, 3, 4, 6, 7
+
+	// Create existing file
+	existingFile := protocol.FileInfo{
+		Name:     "filex",
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[0], blocks[0],
+			blocks[5], blocks[0], blocks[0], blocks[8],
+		},
+	}
+
+	// Create target file
+	requiredFile := existingFile
+	requiredFile.Blocks = blocks[1:]
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	// Update index
+	m.updateLocals("default", []protocol.FileInfo{existingFile})
+
+	p := rwFolder{
+		folder:    "default",
+		dir:       "testdata",
+		model:     m,
+		errors:    make(map[string]string),
+		errorsMut: sync.NewMutex(),
+	}
+
+	copyChan := make(chan copyBlocksState, 1)
+
+	p.handleFile(requiredFile, copyChan, nil)
+
+	// Receive the results
+	toCopy := <-copyChan
+
+	if len(toCopy.blocks) != 8 {
+		t.Errorf("Unexpected count of copy blocks: %d != 8", len(toCopy.blocks))
+	}
+
+	for i, block := range toCopy.blocks {
+		if string(block.Hash) != string(blocks[i+1].Hash) {
+			t.Errorf("Block mismatch: %s != %s", block.String(), blocks[i+1].String())
+		}
+	}
+}
+
+func TestHandleFileWithTemp(t *testing.T) {
+	// After diff between required and existing we should:
+	// Copy: 2, 5, 8
+	// Pull: 1, 3, 4, 6, 7
+
+	// After dropping out blocks already on the temp file we should:
+	// Copy: 5, 8
+	// Pull: 1, 6
+
+	// Create existing file
+	existingFile := protocol.FileInfo{
+		Name:     "file",
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[0], blocks[0],
+			blocks[5], blocks[0], blocks[0], blocks[8],
+		},
+	}
+
+	// Create target file
+	requiredFile := existingFile
+	requiredFile.Blocks = blocks[1:]
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	// Update index
+	m.updateLocals("default", []protocol.FileInfo{existingFile})
+
+	p := rwFolder{
+		folder:    "default",
+		dir:       "testdata",
+		model:     m,
+		errors:    make(map[string]string),
+		errorsMut: sync.NewMutex(),
+	}
+
+	copyChan := make(chan copyBlocksState, 1)
+
+	p.handleFile(requiredFile, copyChan, nil)
+
+	// Receive the results
+	toCopy := <-copyChan
+
+	if len(toCopy.blocks) != 4 {
+		t.Errorf("Unexpected count of copy blocks: %d != 4", len(toCopy.blocks))
+	}
+
+	for i, eq := range []int{1, 5, 6, 8} {
+		if string(toCopy.blocks[i].Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block mismatch: %s != %s", toCopy.blocks[i].String(), blocks[eq].String())
+		}
+	}
+}
+
+func TestCopierFinder(t *testing.T) {
+	// After diff between required and existing we should:
+	// Copy: 1, 2, 3, 4, 6, 7, 8
+	// Since there is no existing file, nor a temp file
+
+	// After dropping out blocks found locally:
+	// Pull: 1, 5, 6, 8
+
+	tempFile := filepath.Join("testdata", defTempNamer.TempName("file2"))
+	err := os.Remove(tempFile)
+	if err != nil && !os.IsNotExist(err) {
+		t.Error(err)
+	}
+
+	// Create existing file
+	existingFile := protocol.FileInfo{
+		Name:     defTempNamer.TempName("file"),
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[3], blocks[4],
+			blocks[0], blocks[0], blocks[7], blocks[0],
+		},
+	}
+
+	// Create target file
+	requiredFile := existingFile
+	requiredFile.Blocks = blocks[1:]
+	requiredFile.Name = "file2"
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+	// Update index
+	m.updateLocals("default", []protocol.FileInfo{existingFile})
+
+	iterFn := func(folder, file string, index int32) bool {
+		return true
+	}
+
+	// Verify that the blocks we say exist on file, really exist in the db.
+	for _, idx := range []int{2, 3, 4, 7} {
+		if m.finder.Iterate(blocks[idx].Hash, iterFn) == false {
+			t.Error("Didn't find block")
+		}
+	}
+
+	p := rwFolder{
+		folder:    "default",
+		dir:       "testdata",
+		model:     m,
+		errors:    make(map[string]string),
+		errorsMut: sync.NewMutex(),
+	}
+
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState, 4)
+	finisherChan := make(chan *sharedPullerState, 1)
+
+	// Run a single fetcher routine
+	go p.copierRoutine(copyChan, pullChan, finisherChan)
+
+	p.handleFile(requiredFile, copyChan, finisherChan)
+
+	pulls := []pullBlockState{<-pullChan, <-pullChan, <-pullChan, <-pullChan}
+	finish := <-finisherChan
+
+	select {
+	case <-pullChan:
+		t.Fatal("Finisher channel has data to be read")
+	case <-finisherChan:
+		t.Fatal("Finisher channel has data to be read")
+	default:
+	}
+
+	// Verify that the right blocks went into the pull list
+	for i, eq := range []int{1, 5, 6, 8} {
+		if string(pulls[i].block.Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, pulls[i].block.String(), blocks[eq].String())
+		}
+		if string(finish.file.Blocks[eq-1].Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, finish.file.Blocks[eq-1].String(), blocks[eq].String())
+		}
+	}
+
+	// Verify that the fetched blocks have actually been written to the temp file
+	blks, err := scanner.HashFile(tempFile, protocol.BlockSize)
+	if err != nil {
+		t.Log(err)
+	}
+
+	for _, eq := range []int{2, 3, 4, 7} {
+		if string(blks[eq-1].Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, blks[eq-1].String(), blocks[eq].String())
+		}
+	}
+	finish.fd.Close()
+
+	os.Remove(tempFile)
+}
+
+// Test that updating a file removes it's old blocks from the blockmap
+func TestCopierCleanup(t *testing.T) {
+	iterFn := func(folder, file string, index int32) bool {
+		return true
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	// Create a file
+	file := protocol.FileInfo{
+		Name:     "test",
+		Flags:    0,
+		Modified: 0,
+		Blocks:   []protocol.BlockInfo{blocks[0]},
+	}
+
+	// Add file to index
+	m.updateLocals("default", []protocol.FileInfo{file})
+
+	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+		t.Error("Expected block not found")
+	}
+
+	file.Blocks = []protocol.BlockInfo{blocks[1]}
+	file.Version = file.Version.Update(protocol.LocalDeviceID.Short())
+	// Update index (removing old blocks)
+	m.updateLocals("default", []protocol.FileInfo{file})
+
+	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+		t.Error("Unexpected block found")
+	}
+
+	if !m.finder.Iterate(blocks[1].Hash, iterFn) {
+		t.Error("Expected block not found")
+	}
+
+	file.Blocks = []protocol.BlockInfo{blocks[0]}
+	file.Version = file.Version.Update(protocol.LocalDeviceID.Short())
+	// Update index (removing old blocks)
+	m.updateLocals("default", []protocol.FileInfo{file})
+
+	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+		t.Error("Unexpected block found")
+	}
+
+	if m.finder.Iterate(blocks[1].Hash, iterFn) {
+		t.Error("Expected block not found")
+	}
+}
+
+// Make sure that the copier routine hashes the content when asked, and pulls
+// if it fails to find the block.
+func TestLastResortPulling(t *testing.T) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	// Add a file to index (with the incorrect block representation, as content
+	// doesn't actually match the block list)
+	file := protocol.FileInfo{
+		Name:     "empty",
+		Flags:    0,
+		Modified: 0,
+		Blocks:   []protocol.BlockInfo{blocks[0]},
+	}
+	m.updateLocals("default", []protocol.FileInfo{file})
+
+	// Pretend that we are handling a new file of the same content but
+	// with a different name (causing to copy that particular block)
+	file.Name = "newfile"
+
+	iterFn := func(folder, file string, index int32) bool {
+		return true
+	}
+
+	// Check that that particular block is there
+	if !m.finder.Iterate(blocks[0].Hash, iterFn) {
+		t.Error("Expected block not found")
+	}
+
+	p := rwFolder{
+		folder:    "default",
+		dir:       "testdata",
+		model:     m,
+		errors:    make(map[string]string),
+		errorsMut: sync.NewMutex(),
+	}
+
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState, 1)
+	finisherChan := make(chan *sharedPullerState, 1)
+
+	// Run a single copier routine
+	go p.copierRoutine(copyChan, pullChan, finisherChan)
+
+	p.handleFile(file, copyChan, finisherChan)
+
+	// Copier should hash empty file, realise that the region it has read
+	// doesn't match the hash which was advertised by the block map, fix it
+	// and ask to pull the block.
+	<-pullChan
+
+	// Verify that it did fix the incorrect hash.
+	if m.finder.Iterate(blocks[0].Hash, iterFn) {
+		t.Error("Found unexpected block")
+	}
+
+	if !m.finder.Iterate(scanner.SHA256OfNothing, iterFn) {
+		t.Error("Expected block not found")
+	}
+
+	(<-finisherChan).fd.Close()
+	os.Remove(filepath.Join("testdata", defTempNamer.TempName("newfile")))
+}
+
+func TestDeregisterOnFailInCopy(t *testing.T) {
+	file := protocol.FileInfo{
+		Name:     "filex",
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[0], blocks[0],
+			blocks[5], blocks[0], blocks[0], blocks[8],
+		},
+	}
+	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	emitter := NewProgressEmitter(defaultConfig)
+	go emitter.Serve()
+
+	p := rwFolder{
+		folder:          "default",
+		dir:             "testdata",
+		model:           m,
+		queue:           newJobQueue(),
+		progressEmitter: emitter,
+		errors:          make(map[string]string),
+		errorsMut:       sync.NewMutex(),
+	}
+
+	// queue.Done should be called by the finisher routine
+	p.queue.Push("filex", 0, 0)
+	p.queue.Pop()
+
+	if len(p.queue.progress) != 1 {
+		t.Fatal("Expected file in progress")
+	}
+
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState)
+	finisherBufferChan := make(chan *sharedPullerState)
+	finisherChan := make(chan *sharedPullerState)
+
+	go p.copierRoutine(copyChan, pullChan, finisherBufferChan)
+	go p.finisherRoutine(finisherChan)
+
+	p.handleFile(file, copyChan, finisherChan)
+
+	// Receive a block at puller, to indicate that at least a single copier
+	// loop has been performed.
+	toPull := <-pullChan
+	// Wait until copier is trying to pass something down to the puller again
+	time.Sleep(100 * time.Millisecond)
+	// Close the file
+	toPull.sharedPullerState.fail("test", os.ErrNotExist)
+	// Unblock copier
+	<-pullChan
+
+	select {
+	case state := <-finisherBufferChan:
+		// At this point the file should still be registered with both the job
+		// queue, and the progress emitter. Verify this.
+		if len(p.progressEmitter.registry) != 1 || len(p.queue.progress) != 1 || len(p.queue.queued) != 0 {
+			t.Fatal("Could not find file")
+		}
+
+		// Pass the file down the real finisher, and give it time to consume
+		finisherChan <- state
+		time.Sleep(100 * time.Millisecond)
+
+		if state.fd != nil {
+			t.Fatal("File not closed?")
+		}
+
+		if len(p.progressEmitter.registry) != 0 || len(p.queue.progress) != 0 || len(p.queue.queued) != 0 {
+			t.Fatal("Still registered", len(p.progressEmitter.registry), len(p.queue.progress), len(p.queue.queued))
+		}
+
+		// Doing it again should have no effect
+		finisherChan <- state
+		time.Sleep(100 * time.Millisecond)
+
+		if len(p.progressEmitter.registry) != 0 || len(p.queue.progress) != 0 || len(p.queue.queued) != 0 {
+			t.Fatal("Still registered")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Didn't get anything to the finisher")
+	}
+}
+
+func TestDeregisterOnFailInPull(t *testing.T) {
+	file := protocol.FileInfo{
+		Name:     "filex",
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[0], blocks[0],
+			blocks[5], blocks[0], blocks[0], blocks[8],
+		},
+	}
+	defer os.Remove("testdata/" + defTempNamer.TempName("filex"))
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db)
+	m.AddFolder(defaultFolderConfig)
+
+	emitter := NewProgressEmitter(defaultConfig)
+	go emitter.Serve()
+
+	p := rwFolder{
+		folder:          "default",
+		dir:             "testdata",
+		model:           m,
+		queue:           newJobQueue(),
+		progressEmitter: emitter,
+		errors:          make(map[string]string),
+		errorsMut:       sync.NewMutex(),
+	}
+
+	// queue.Done should be called by the finisher routine
+	p.queue.Push("filex", 0, 0)
+	p.queue.Pop()
+
+	if len(p.queue.progress) != 1 {
+		t.Fatal("Expected file in progress")
+	}
+
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState)
+	finisherBufferChan := make(chan *sharedPullerState)
+	finisherChan := make(chan *sharedPullerState)
+
+	go p.copierRoutine(copyChan, pullChan, finisherBufferChan)
+	go p.pullerRoutine(pullChan, finisherBufferChan)
+	go p.finisherRoutine(finisherChan)
+
+	p.handleFile(file, copyChan, finisherChan)
+
+	// Receove at finisher, we shoud error out as puller has nowhere to pull
+	// from.
+	select {
+	case state := <-finisherBufferChan:
+		// At this point the file should still be registered with both the job
+		// queue, and the progress emitter. Verify this.
+		if len(p.progressEmitter.registry) != 1 || len(p.queue.progress) != 1 || len(p.queue.queued) != 0 {
+			t.Fatal("Could not find file")
+		}
+
+		// Pass the file down the real finisher, and give it time to consume
+		finisherChan <- state
+		time.Sleep(100 * time.Millisecond)
+
+		if state.fd != nil {
+			t.Fatal("File not closed?")
+		}
+
+		if len(p.progressEmitter.registry) != 0 || len(p.queue.progress) != 0 || len(p.queue.queued) != 0 {
+			t.Fatal("Still registered", len(p.progressEmitter.registry), len(p.queue.progress), len(p.queue.queued))
+		}
+
+		// Doing it again should have no effect
+		finisherChan <- state
+		time.Sleep(100 * time.Millisecond)
+
+		if len(p.progressEmitter.registry) != 0 || len(p.queue.progress) != 0 || len(p.queue.queued) != 0 {
+			t.Fatal("Still registered")
+		}
+	case <-time.After(time.Second):
+		t.Fatal("Didn't get anything to the finisher")
+	}
+}

lib/model/sharedpullerstate.go

@@ -0,0 +1,262 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+// A sharedPullerState is kept for each file that is being synced and is kept
+// updated along the way.
+type sharedPullerState struct {
+	// Immutable, does not require locking
+	file        protocol.FileInfo // The new file (desired end state)
+	folder      string
+	tempName    string
+	realName    string
+	reused      int // Number of blocks reused from temporary file
+	ignorePerms bool
+	version     protocol.Vector // The current (old) version
+
+	// Mutable, must be locked for access
+	err        error      // The first error we hit
+	fd         *os.File   // The fd of the temp file
+	copyTotal  int        // Total number of copy actions for the whole job
+	pullTotal  int        // Total number of pull actions for the whole job
+	copyOrigin int        // Number of blocks copied from the original file
+	copyNeeded int        // Number of copy actions still pending
+	pullNeeded int        // Number of block pulls still pending
+	closed     bool       // True if the file has been finalClosed.
+	mut        sync.Mutex // Protects the above
+}
+
+// A momentary state representing the progress of the puller
+type pullerProgress struct {
+	Total               int   `json:"total"`
+	Reused              int   `json:"reused"`
+	CopiedFromOrigin    int   `json:"copiedFromOrigin"`
+	CopiedFromElsewhere int   `json:"copiedFromElsewhere"`
+	Pulled              int   `json:"pulled"`
+	Pulling             int   `json:"pulling"`
+	BytesDone           int64 `json:"bytesDone"`
+	BytesTotal          int64 `json:"bytesTotal"`
+}
+
+// A lockedWriterAt synchronizes WriteAt calls with an external mutex.
+// WriteAt() is goroutine safe by itself, but not against for example Close().
+type lockedWriterAt struct {
+	mut *sync.Mutex
+	wr  io.WriterAt
+}
+
+func (w lockedWriterAt) WriteAt(p []byte, off int64) (n int, err error) {
+	(*w.mut).Lock()
+	defer (*w.mut).Unlock()
+	return w.wr.WriteAt(p, off)
+}
+
+// tempFile returns the fd for the temporary file, reusing an open fd
+// or creating the file as necessary.
+func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	// If we've already hit an error, return early
+	if s.err != nil {
+		return nil, s.err
+	}
+
+	// If the temp file is already open, return the file descriptor
+	if s.fd != nil {
+		return lockedWriterAt{&s.mut, s.fd}, nil
+	}
+
+	// Ensure that the parent directory is writable. This is
+	// osutil.InWritableDir except we need to do more stuff so we duplicate it
+	// here.
+	dir := filepath.Dir(s.tempName)
+	if info, err := os.Stat(dir); err != nil {
+		s.failLocked("dst stat dir", err)
+		return nil, err
+	} else if info.Mode()&0200 == 0 {
+		err := os.Chmod(dir, 0755)
+		if !s.ignorePerms && err == nil {
+			defer func() {
+				err := os.Chmod(dir, info.Mode().Perm())
+				if err != nil {
+					panic(err)
+				}
+			}()
+		}
+	}
+
+	// Attempt to create the temp file
+	flags := os.O_WRONLY
+	if s.reused == 0 {
+		flags |= os.O_CREATE | os.O_EXCL
+	} else {
+		// With sufficiently bad luck when exiting or crashing, we may have
+		// had time to chmod the temp file to read only state but not yet
+		// moved it to it's final name. This leaves us with a read only temp
+		// file that we're going to try to reuse. To handle that, we need to
+		// make sure we have write permissions on the file before opening it.
+		err := os.Chmod(s.tempName, 0644)
+		if !s.ignorePerms && err != nil {
+			s.failLocked("dst create chmod", err)
+			return nil, err
+		}
+	}
+	fd, err := os.OpenFile(s.tempName, flags, 0666)
+	if err != nil {
+		s.failLocked("dst create", err)
+		return nil, err
+	}
+
+	// Same fd will be used by all writers
+	s.fd = fd
+
+	return lockedWriterAt{&s.mut, s.fd}, nil
+}
+
+// sourceFile opens the existing source file for reading
+func (s *sharedPullerState) sourceFile() (*os.File, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	// If we've already hit an error, return early
+	if s.err != nil {
+		return nil, s.err
+	}
+
+	// Attempt to open the existing file
+	fd, err := os.Open(s.realName)
+	if err != nil {
+		s.failLocked("src open", err)
+		return nil, err
+	}
+
+	return fd, nil
+}
+
+// earlyClose prints a warning message composed of the context and
+// error, and marks the sharedPullerState as failed. Is a no-op when called on
+// an already failed state.
+func (s *sharedPullerState) fail(context string, err error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	s.failLocked(context, err)
+}
+
+func (s *sharedPullerState) failLocked(context string, err error) {
+	if s.err != nil {
+		return
+	}
+
+	l.Infof("Puller (folder %q, file %q): %s: %v", s.folder, s.file.Name, context, err)
+	s.err = err
+}
+
+func (s *sharedPullerState) failed() error {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	return s.err
+}
+
+func (s *sharedPullerState) copyDone() {
+	s.mut.Lock()
+	s.copyNeeded--
+	if debug {
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
+	}
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) copiedFromOrigin() {
+	s.mut.Lock()
+	s.copyOrigin++
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) pullStarted() {
+	s.mut.Lock()
+	s.copyTotal--
+	s.copyNeeded--
+	s.pullTotal++
+	s.pullNeeded++
+	if debug {
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded start ->", s.pullNeeded)
+	}
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) pullDone() {
+	s.mut.Lock()
+	s.pullNeeded--
+	if debug {
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
+	}
+	s.mut.Unlock()
+}
+
+// finalClose atomically closes and returns closed status of a file. A true
+// first return value means the file was closed and should be finished, with
+// the error indicating the success or failure of the close. A false first
+// return value indicates the file is not ready to be closed, or is already
+// closed and should in either case not be finished off now.
+func (s *sharedPullerState) finalClose() (bool, error) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	if s.closed {
+		// Already closed
+		return false, nil
+	}
+
+	if s.pullNeeded+s.copyNeeded != 0 && s.err == nil {
+		// Not done yet, and not errored
+		return false, nil
+	}
+
+	if s.fd != nil {
+		if closeErr := s.fd.Close(); closeErr != nil && s.err == nil {
+			// This is our error if we weren't errored before. Otherwise we
+			// keep the earlier error.
+			s.err = closeErr
+		}
+		s.fd = nil
+	}
+
+	s.closed = true
+
+	return true, s.err
+}
+
+// Returns the momentarily progress for the puller
+func (s *sharedPullerState) Progress() *pullerProgress {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+	total := s.reused + s.copyTotal + s.pullTotal
+	done := total - s.copyNeeded - s.pullNeeded
+	return &pullerProgress{
+		Total:               total,
+		Reused:              s.reused,
+		CopiedFromOrigin:    s.copyOrigin,
+		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
+		Pulled:              s.pullTotal - s.pullNeeded,
+		Pulling:             s.pullNeeded,
+		BytesTotal:          db.BlocksToSize(total),
+		BytesDone:           db.BlocksToSize(done),
+	}
+}

lib/model/sharedpullerstate_test.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func TestSourceFileOK(t *testing.T) {
+	s := sharedPullerState{
+		realName: "testdata/foo",
+		mut:      sync.NewMutex(),
+	}
+
+	fd, err := s.sourceFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fd == nil {
+		t.Fatal("Unexpected nil fd")
+	}
+
+	bs := make([]byte, 6)
+	n, err := fd.Read(bs)
+
+	if n != len(bs) {
+		t.Fatalf("Wrong read length %d != %d", n, len(bs))
+	}
+	if string(bs) != "foobar" {
+		t.Fatalf("Wrong contents %s != foobar", string(bs))
+	}
+
+	if err := s.failed(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSourceFileBad(t *testing.T) {
+	s := sharedPullerState{
+		realName: "nonexistent",
+		mut:      sync.NewMutex(),
+	}
+
+	fd, err := s.sourceFile()
+	if err == nil {
+		t.Fatal("Unexpected nil error")
+	}
+	if fd != nil {
+		t.Fatal("Unexpected non-nil fd")
+	}
+	if err := s.failed(); err == nil {
+		t.Fatal("Unexpected nil failed()")
+	}
+}
+
+// Test creating temporary file inside read-only directory
+func TestReadOnlyDir(t *testing.T) {
+	// Create a read only directory, clean it up afterwards.
+	os.Mkdir("testdata/read_only_dir", 0555)
+	defer func() {
+		os.Chmod("testdata/read_only_dir", 0755)
+		os.RemoveAll("testdata/read_only_dir")
+	}()
+
+	s := sharedPullerState{
+		tempName: "testdata/read_only_dir/.temp_name",
+		mut:      sync.NewMutex(),
+	}
+
+	fd, err := s.tempFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fd == nil {
+		t.Fatal("Unexpected nil fd")
+	}
+
+	s.fail("Test done", nil)
+	s.finalClose()
+}

lib/model/tempname.go

@@ -0,0 +1,45 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"crypto/md5"
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type tempNamer struct {
+	prefix string
+}
+
+var defTempNamer tempNamer
+
+func init() {
+	if runtime.GOOS == "windows" {
+		defTempNamer = tempNamer{"~syncthing~"}
+	} else {
+		defTempNamer = tempNamer{".syncthing."}
+	}
+}
+
+func (t tempNamer) IsTemporary(name string) bool {
+	return strings.HasPrefix(filepath.Base(name), t.prefix)
+}
+
+func (t tempNamer) TempName(name string) string {
+	tdir := filepath.Dir(name)
+	tbase := filepath.Base(name)
+	if len(tbase) > 240 {
+		hash := md5.New()
+		hash.Write([]byte(name))
+		tbase = fmt.Sprintf("%x", hash.Sum(nil))
+	}
+	tname := fmt.Sprintf("%s%s.tmp", t.prefix, tbase)
+	return filepath.Join(tdir, tname)
+}

lib/model/tempname_test.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestLongTempFilename(t *testing.T) {
+	filename := ""
+	for i := 0; i < 300; i++ {
+		filename += "l"
+	}
+	tFile := defTempNamer.TempName(filename)
+	if len(tFile) < 10 || len(tFile) > 200 {
+		t.Fatal("Invalid long filename")
+	}
+	if !strings.HasSuffix(defTempNamer.TempName("short"), "short.tmp") {
+		t.Fatal("Invalid short filename", defTempNamer.TempName("short"))
+	}
+}

lib/model/testdata/.stignore

@@ -0,0 +1,2 @@
+.*
+quux

lib/model/testdata/bar

@@ -0,0 +1 @@
+foobarbaz

lib/model/testdata/baz/quux

@@ -0,0 +1 @@
+baazquux

lib/model/testdata/foo

@@ -0,0 +1 @@
+foobar

lib/model/util.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"sync"
+	"time"
+)
+
+func deadlockDetect(mut sync.Locker, timeout time.Duration) {
+	go func() {
+		for {
+			time.Sleep(timeout / 4)
+			ok := make(chan bool, 2)
+
+			go func() {
+				mut.Lock()
+				mut.Unlock()
+				ok <- true
+			}()
+
+			go func() {
+				time.Sleep(timeout)
+				ok <- false
+			}()
+
+			if r := <-ok; !r {
+				panic("deadlock detected")
+			}
+		}
+	}()
+}

lib/osutil/atomic.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+var (
+	ErrClosed  = errors.New("write to closed writer")
+	TempPrefix = ".syncthing.tmp."
+)
+
+// An AtomicWriter is an *os.File that writes to a temporary file in the same
+// directory as the final path. On successfull Close the file is renamed to
+// it's final path. Any error on Write or during Close is accumulated and
+// returned on Close, so a lazy user can ignore errors until Close.
+type AtomicWriter struct {
+	path string
+	next *os.File
+	err  error
+}
+
+// CreateAtomic is like os.Create with a FileMode, except a temporary file
+// name is used instead of the given name.
+func CreateAtomic(path string, mode os.FileMode) (*AtomicWriter, error) {
+	fd, err := ioutil.TempFile(filepath.Dir(path), TempPrefix)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := os.Chmod(fd.Name(), mode); err != nil {
+		fd.Close()
+		os.Remove(fd.Name())
+		return nil, err
+	}
+
+	w := &AtomicWriter{
+		path: path,
+		next: fd,
+	}
+
+	return w, nil
+}
+
+// Write is like io.Writer, but is a no-op on an already failed AtomicWriter.
+func (w *AtomicWriter) Write(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	n, err := w.next.Write(bs)
+	if err != nil {
+		w.err = err
+		w.next.Close()
+	}
+	return n, err
+}
+
+// Close closes the temporary file and renames it to the final path. It is
+// invalid to call Write() or Close() after Close().
+func (w *AtomicWriter) Close() error {
+	if w.err != nil {
+		return w.err
+	}
+
+	// Try to not leave temp file around, but ignore error.
+	defer os.Remove(w.next.Name())
+
+	if err := w.next.Close(); err != nil {
+		w.err = err
+		return err
+	}
+
+	// Remove the destination file, on Windows only. If it fails, and not due
+	// to the file not existing, we won't be able to complete the rename
+	// either. Return this error because it may be more informative. On non-
+	// Windows we want the atomic rename behavior so we don't attempt remove.
+	if runtime.GOOS == "windows" {
+		if err := os.Remove(w.path); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	if err := os.Rename(w.next.Name(), w.path); err != nil {
+		w.err = err
+		return err
+	}
+
+	// Set w.err to return appropriately for any future operations.
+	w.err = ErrClosed
+
+	return nil
+}

lib/osutil/atomic_test.go

@@ -0,0 +1,85 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestCreateAtomicCreate(t *testing.T) {
+	os.RemoveAll("testdata")
+	defer os.RemoveAll("testdata")
+
+	if err := os.Mkdir("testdata", 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	w, err := CreateAtomic("testdata/file", 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	n, err := w.Write([]byte("hello"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 5 {
+		t.Fatal("written bytes", n, "!= 5")
+	}
+
+	if _, err := ioutil.ReadFile("testdata/file"); err == nil {
+		t.Fatal("file should not exist")
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err := ioutil.ReadFile("testdata/file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bs, []byte("hello")) {
+		t.Error("incorrect data")
+	}
+}
+
+func TestCreateAtomicReplace(t *testing.T) {
+	os.RemoveAll("testdata")
+	defer os.RemoveAll("testdata")
+
+	if err := os.Mkdir("testdata", 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile("testdata/file", []byte("some old data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	w, err := CreateAtomic("testdata/file", 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := w.Write([]byte("hello")); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err := ioutil.ReadFile("testdata/file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(bs, []byte("hello")) {
+		t.Error("incorrect data")
+	}
+}

lib/osutil/filenames_darwin.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import "golang.org/x/text/unicode/norm"
+
+func NormalizedFilename(s string) string {
+	return norm.NFC.String(s)
+}
+
+func NativeFilename(s string) string {
+	return norm.NFD.String(s)
+}