Move top level packages to internal.

internal/auto/auto_test.go

@@ -0,0 +1,23 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package auto_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/auto"
+)
+
+func TestAssets(t *testing.T) {
+	assets := auto.Assets()
+	idx, ok := assets["index.html"]
+	if !ok {
+		t.Fatal("No index.html in compiled in assets")
+	}
+	if !bytes.Contains(idx, []byte("<html")) {
+		t.Fatal("No html in index.html")
+	}
+}

internal/auto/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package auto contains auto generated files for web assets.
+package auto

internal/beacon/beacon.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import "net"
+
+type recv struct {
+	data []byte
+	src  net.Addr
+}
+
+type Interface interface {
+	Send(data []byte)
+	Recv() ([]byte, net.Addr)
+}
+
+func genericReader(conn *net.UDPConn, outbox chan<- recv) {
+	bs := make([]byte, 65536)
+	for {
+		n, addr, err := conn.ReadFrom(bs)
+		if err != nil {
+			l.Warnln("multicast read:", err)
+			return
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
+
+		c := make([]byte, n)
+		copy(c, bs)
+		select {
+		case outbox <- recv{c, addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
+			}
+		}
+	}
+}

internal/beacon/broadcast.go

@@ -0,0 +1,97 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import "net"
+
+type Broadcast struct {
+	conn   *net.UDPConn
+	port   int
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewBroadcast(port int) (*Broadcast, error) {
+	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, err
+	}
+	b := &Broadcast{
+		conn:   conn,
+		port:   port,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go genericReader(b.conn, b.outbox)
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Broadcast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Broadcast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Broadcast) writer() {
+	for bs := range b.inbox {
+
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			l.Warnln("Broadcast: interface addresses:", err)
+			continue
+		}
+
+		var dsts []net.IP
+		for _, addr := range addrs {
+			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
+				baddr := bcast(iaddr)
+				dsts = append(dsts, baddr.IP)
+			}
+		}
+
+		if len(dsts) == 0 {
+			// Fall back to the general IPv4 broadcast address
+			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
+		}
+
+		if debug {
+			l.Debugln("addresses:", dsts)
+		}
+
+		for _, ip := range dsts {
+			dst := &net.UDPAddr{IP: ip, Port: b.port}
+
+			_, err := b.conn.WriteTo(bs, dst)
+			if err != nil {
+				if debug {
+					l.Debugln(err)
+				}
+			} else if debug {
+				l.Debugf("sent %d bytes to %s", len(bs), dst)
+			}
+		}
+	}
+}
+
+func bcast(ip *net.IPNet) *net.IPNet {
+	var bc = &net.IPNet{}
+	bc.IP = make([]byte, len(ip.IP))
+	copy(bc.IP, ip.IP)
+	bc.Mask = ip.Mask
+
+	offset := len(bc.IP) - len(bc.Mask)
+	for i := range bc.IP {
+		if i-offset >= 0 {
+			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
+		}
+	}
+	return bc
+}

internal/beacon/broadcast_test.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import (
+	"net"
+	"testing"
+)
+
+var addrToBcast = []struct {
+	addr, bcast string
+}{
+	{"172.16.32.33/25", "172.16.32.127/25"},
+	{"172.16.32.129/25", "172.16.32.255/25"},
+	{"172.16.32.33/24", "172.16.32.255/24"},
+	{"172.16.32.33/22", "172.16.35.255/22"},
+	{"172.16.32.33/0", "255.255.255.255/0"},
+	{"172.16.32.33/32", "172.16.32.33/32"},
+}
+
+func TestBroadcastAddr(t *testing.T) {
+	for _, tc := range addrToBcast {
+		_, net, err := net.ParseCIDR(tc.addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bc := bcast(net).String()
+		if bc != tc.bcast {
+			t.Errorf("%q != %q", bc, tc.bcast)
+		}
+	}
+}

internal/beacon/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/beacon/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package beacon implements an UDP broadcast beacon
+package beacon

internal/beacon/multicast.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package beacon
+
+import "net"
+
+type Multicast struct {
+	conn   *net.UDPConn
+	addr   *net.UDPAddr
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func NewMulticast(addr string) (*Multicast, error) {
+	gaddr, err := net.ResolveUDPAddr("udp", addr)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := net.ListenMulticastUDP("udp", nil, gaddr)
+	if err != nil {
+		return nil, err
+	}
+	b := &Multicast{
+		conn:   conn,
+		addr:   gaddr,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go genericReader(b.conn, b.outbox)
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Multicast) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Multicast) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Multicast) writer() {
+	for bs := range b.inbox {
+		intfs, err := net.Interfaces()
+		if err != nil {
+			l.Warnln("multicast interfaces:", err)
+			continue
+		}
+		for _, intf := range intfs {
+			if intf.Flags&net.FlagUp != 0 && intf.Flags&net.FlagMulticast != 0 {
+				addr := *b.addr
+				addr.Zone = intf.Name
+				_, err = b.conn.WriteTo(bs, &addr)
+				if err != nil {
+					if debug {
+						l.Debugln(err, "on write to", addr)
+					}
+				} else if debug {
+					l.Debugf("sent %d bytes to %s", len(bs), addr.String())
+				}
+			}
+		}
+	}
+}

internal/config/config.go

@@ -0,0 +1,616 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package config implements reading and writing of the syncthing configuration file.
+package config
+
+import (
+	"encoding/xml"
+	"fmt"
+	"os"
+	"reflect"
+	"sort"
+	"strconv"
+
+	"code.google.com/p/go.crypto/bcrypt"
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/logger"
+	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+var l = logger.DefaultLogger
+
+type Configuration struct {
+	Location     string                    `xml:"-" json:"-"`
+	Version      int                       `xml:"version,attr" default:"3"`
+	Repositories []RepositoryConfiguration `xml:"repository"`
+	Nodes        []NodeConfiguration       `xml:"node"`
+	GUI          GUIConfiguration          `xml:"gui"`
+	Options      OptionsConfiguration      `xml:"options"`
+	XMLName      xml.Name                  `xml:"configuration" json:"-"`
+}
+
+type RepositoryConfiguration struct {
+	ID              string                        `xml:"id,attr"`
+	Directory       string                        `xml:"directory,attr"`
+	Nodes           []RepositoryNodeConfiguration `xml:"node"`
+	ReadOnly        bool                          `xml:"ro,attr"`
+	RescanIntervalS int                           `xml:"rescanIntervalS,attr" default:"60"`
+	IgnorePerms     bool                          `xml:"ignorePerms,attr"`
+	Invalid         string                        `xml:"-"` // Set at runtime when there is an error, not saved
+	Versioning      VersioningConfiguration       `xml:"versioning"`
+
+	nodeIDs []protocol.NodeID
+}
+
+type VersioningConfiguration struct {
+	Type   string `xml:"type,attr"`
+	Params map[string]string
+}
+
+type InternalVersioningConfiguration struct {
+	Type   string          `xml:"type,attr,omitempty"`
+	Params []InternalParam `xml:"param"`
+}
+
+type InternalParam struct {
+	Key string `xml:"key,attr"`
+	Val string `xml:"val,attr"`
+}
+
+func (c *VersioningConfiguration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	tmp.Type = c.Type
+	for k, v := range c.Params {
+		tmp.Params = append(tmp.Params, InternalParam{k, v})
+	}
+
+	return e.EncodeElement(tmp, start)
+
+}
+
+func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var tmp InternalVersioningConfiguration
+	err := d.DecodeElement(&tmp, &start)
+	if err != nil {
+		return err
+	}
+
+	c.Type = tmp.Type
+	c.Params = make(map[string]string, len(tmp.Params))
+	for _, p := range tmp.Params {
+		c.Params[p.Key] = p.Val
+	}
+	return nil
+}
+
+func (r *RepositoryConfiguration) NodeIDs() []protocol.NodeID {
+	if r.nodeIDs == nil {
+		for _, n := range r.Nodes {
+			r.nodeIDs = append(r.nodeIDs, n.NodeID)
+		}
+	}
+	return r.nodeIDs
+}
+
+type NodeConfiguration struct {
+	NodeID      protocol.NodeID `xml:"id,attr"`
+	Name        string          `xml:"name,attr,omitempty"`
+	Addresses   []string        `xml:"address,omitempty"`
+	Compression bool            `xml:"compression,attr"`
+	CertName    string          `xml:"certName,attr,omitempty"`
+	Introducer  bool            `xml:"introducer,attr"`
+}
+
+type RepositoryNodeConfiguration struct {
+	NodeID protocol.NodeID `xml:"id,attr"`
+
+	Deprecated_Name      string   `xml:"name,attr,omitempty" json:"-"`
+	Deprecated_Addresses []string `xml:"address,omitempty" json:"-"`
+}
+
+type OptionsConfiguration struct {
+	ListenAddress      []string `xml:"listenAddress" default:"0.0.0.0:22000"`
+	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
+	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
+	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
+	LocalAnnPort       int      `xml:"localAnnouncePort" default:"21025"`
+	LocalAnnMCAddr     string   `xml:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
+	ParallelRequests   int      `xml:"parallelRequests" default:"16"`
+	MaxSendKbps        int      `xml:"maxSendKbps"`
+	MaxRecvKbps        int      `xml:"maxRecvKbps"`
+	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
+	StartBrowser       bool     `xml:"startBrowser" default:"true"`
+	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
+	UPnPLease          int      `xml:"upnpLeaseMinutes" default:"0"`
+	UPnPRenewal        int      `xml:"upnpRenewalMinutes" default:"30"`
+	URAccepted         int      `xml:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	RestartOnWakeup    bool     `xml:"restartOnWakeup" default:"true"`
+
+	Deprecated_RescanIntervalS int    `xml:"rescanIntervalS,omitempty" json:"-"`
+	Deprecated_UREnabled       bool   `xml:"urEnabled,omitempty" json:"-"`
+	Deprecated_URDeclined      bool   `xml:"urDeclined,omitempty" json:"-"`
+	Deprecated_ReadOnly        bool   `xml:"readOnly,omitempty" json:"-"`
+	Deprecated_GUIEnabled      bool   `xml:"guiEnabled,omitempty" json:"-"`
+	Deprecated_GUIAddress      string `xml:"guiAddress,omitempty" json:"-"`
+}
+
+type GUIConfiguration struct {
+	Enabled  bool   `xml:"enabled,attr" default:"true"`
+	Address  string `xml:"address" default:"127.0.0.1:8080"`
+	User     string `xml:"user,omitempty"`
+	Password string `xml:"password,omitempty"`
+	UseTLS   bool   `xml:"tls,attr"`
+	APIKey   string `xml:"apikey,omitempty"`
+}
+
+func (cfg *Configuration) NodeMap() map[protocol.NodeID]NodeConfiguration {
+	m := make(map[protocol.NodeID]NodeConfiguration, len(cfg.Nodes))
+	for _, n := range cfg.Nodes {
+		m[n.NodeID] = n
+	}
+	return m
+}
+
+func (cfg *Configuration) GetNodeConfiguration(nodeID protocol.NodeID) *NodeConfiguration {
+	for i, node := range cfg.Nodes {
+		if node.NodeID == nodeID {
+			return &cfg.Nodes[i]
+		}
+	}
+	return nil
+}
+
+func (cfg *Configuration) GetRepoConfiguration(repoID string) *RepositoryConfiguration {
+	for i, repo := range cfg.Repositories {
+		if repo.ID == repoID {
+			return &cfg.Repositories[i]
+		}
+	}
+	return nil
+}
+
+func (cfg *Configuration) RepoMap() map[string]RepositoryConfiguration {
+	m := make(map[string]RepositoryConfiguration, len(cfg.Repositories))
+	for _, r := range cfg.Repositories {
+		m[r.ID] = r
+	}
+	return m
+}
+
+func setDefaults(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case string:
+				f.SetString(v)
+
+			case int:
+				i, err := strconv.ParseInt(v, 10, 64)
+				if err != nil {
+					return err
+				}
+				f.SetInt(i)
+
+			case bool:
+				f.SetBool(v == "true")
+
+			case []string:
+				// We don't do anything with string slices here. Any default
+				// we set will be appended to by the XML decoder, so we fill
+				// those after decoding.
+
+			default:
+				panic(f.Type())
+			}
+		}
+	}
+	return nil
+}
+
+// fillNilSlices sets default value on slices that are still nil.
+func fillNilSlices(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case []string:
+				if f.IsNil() {
+					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
+					rv.Index(0).SetString(v)
+					f.Set(rv)
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (cfg *Configuration) Save() error {
+	fd, err := os.Create(cfg.Location + ".tmp")
+	if err != nil {
+		l.Warnln("Saving config:", err)
+		return err
+	}
+
+	e := xml.NewEncoder(fd)
+	e.Indent("", "    ")
+	err = e.Encode(cfg)
+	if err != nil {
+		fd.Close()
+		return err
+	}
+	_, err = fd.Write([]byte("\n"))
+
+	if err != nil {
+		l.Warnln("Saving config:", err)
+		fd.Close()
+		return err
+	}
+
+	err = fd.Close()
+	if err != nil {
+		l.Warnln("Saving config:", err)
+		return err
+	}
+
+	err = osutil.Rename(cfg.Location+".tmp", cfg.Location)
+	if err != nil {
+		l.Warnln("Saving config:", err)
+	}
+	events.Default.Log(events.ConfigSaved, cfg)
+	return err
+}
+
+func uniqueStrings(ss []string) []string {
+	var m = make(map[string]bool, len(ss))
+	for _, s := range ss {
+		m[s] = true
+	}
+
+	var us = make([]string, 0, len(m))
+	for k := range m {
+		us = append(us, k)
+	}
+
+	return us
+}
+
+func (cfg *Configuration) prepare(myID protocol.NodeID) {
+	fillNilSlices(&cfg.Options)
+
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+
+	// Initialize an empty slice for repositories if the config has none
+	if cfg.Repositories == nil {
+		cfg.Repositories = []RepositoryConfiguration{}
+	}
+
+	// Check for missing, bad or duplicate repository ID:s
+	var seenRepos = map[string]*RepositoryConfiguration{}
+	var uniqueCounter int
+	for i := range cfg.Repositories {
+		repo := &cfg.Repositories[i]
+
+		if len(repo.Directory) == 0 {
+			repo.Invalid = "no directory configured"
+			continue
+		}
+
+		if repo.ID == "" {
+			repo.ID = "default"
+		}
+
+		if seen, ok := seenRepos[repo.ID]; ok {
+			l.Warnf("Multiple repositories with ID %q; disabling", repo.ID)
+
+			seen.Invalid = "duplicate repository ID"
+			if seen.ID == repo.ID {
+				uniqueCounter++
+				seen.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
+			}
+			repo.Invalid = "duplicate repository ID"
+			uniqueCounter++
+			repo.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
+		} else {
+			seenRepos[repo.ID] = repo
+		}
+	}
+
+	if cfg.Options.Deprecated_URDeclined {
+		cfg.Options.URAccepted = -1
+	}
+	cfg.Options.Deprecated_URDeclined = false
+	cfg.Options.Deprecated_UREnabled = false
+
+	// Upgrade to v2 configuration if appropriate
+	if cfg.Version == 1 {
+		convertV1V2(cfg)
+	}
+
+	// Upgrade to v3 configuration if appropriate
+	if cfg.Version == 2 {
+		convertV2V3(cfg)
+	}
+
+	// Upgrade to v4 configuration if appropriate
+	if cfg.Version == 3 {
+		convertV3V4(cfg)
+	}
+
+	// Hash old cleartext passwords
+	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
+		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
+		if err != nil {
+			l.Warnln("bcrypting password:", err)
+		} else {
+			cfg.GUI.Password = string(hash)
+		}
+	}
+
+	// Build a list of available nodes
+	existingNodes := make(map[protocol.NodeID]bool)
+	existingNodes[myID] = true
+	for _, node := range cfg.Nodes {
+		existingNodes[node.NodeID] = true
+	}
+
+	// Ensure this node is present in all relevant places
+	me := cfg.GetNodeConfiguration(myID)
+	if me == nil {
+		myName, _ := os.Hostname()
+		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
+			NodeID: myID,
+			Name:   myName,
+		})
+	}
+	sort.Sort(NodeConfigurationList(cfg.Nodes))
+	// Ensure that any loose nodes are not present in the wrong places
+	// Ensure that there are no duplicate nodes
+	for i := range cfg.Repositories {
+		cfg.Repositories[i].Nodes = ensureNodePresent(cfg.Repositories[i].Nodes, myID)
+		cfg.Repositories[i].Nodes = ensureExistingNodes(cfg.Repositories[i].Nodes, existingNodes)
+		cfg.Repositories[i].Nodes = ensureNoDuplicates(cfg.Repositories[i].Nodes)
+		sort.Sort(RepositoryNodeConfigurationList(cfg.Repositories[i].Nodes))
+	}
+
+	// An empty address list is equivalent to a single "dynamic" entry
+	for i := range cfg.Nodes {
+		n := &cfg.Nodes[i]
+		if len(n.Addresses) == 0 || len(n.Addresses) == 1 && n.Addresses[0] == "" {
+			n.Addresses = []string{"dynamic"}
+		}
+	}
+}
+
+func New(location string, myID protocol.NodeID) Configuration {
+	var cfg Configuration
+
+	cfg.Location = location
+
+	setDefaults(&cfg)
+	setDefaults(&cfg.Options)
+	setDefaults(&cfg.GUI)
+
+	cfg.prepare(myID)
+
+	return cfg
+}
+
+func Load(location string, myID protocol.NodeID) (Configuration, error) {
+	var cfg Configuration
+
+	cfg.Location = location
+
+	setDefaults(&cfg)
+	setDefaults(&cfg.Options)
+	setDefaults(&cfg.GUI)
+
+	fd, err := os.Open(location)
+	if err != nil {
+		return Configuration{}, err
+	}
+	err = xml.NewDecoder(fd).Decode(&cfg)
+	fd.Close()
+
+	cfg.prepare(myID)
+
+	return cfg, err
+}
+
+// ChangeRequiresRestart returns true if updating the configuration requires a
+// complete restart.
+func ChangeRequiresRestart(from, to Configuration) bool {
+	// Adding, removing or changing repos requires restart
+	if len(from.Repositories) != len(to.Repositories) {
+		return true
+	}
+	fromRepos := from.RepoMap()
+	toRepos := to.RepoMap()
+	for id := range fromRepos {
+		if !reflect.DeepEqual(fromRepos[id], toRepos[id]) {
+			return true
+		}
+	}
+
+	// Removing a node requires a restart. Adding one does not. Changing
+	// address or name does not.
+	fromNodes := from.NodeMap()
+	toNodes := to.NodeMap()
+	for nodeID := range fromNodes {
+		if _, ok := toNodes[nodeID]; !ok {
+			return true
+		}
+	}
+
+	// All of the generic options require restart
+	if !reflect.DeepEqual(from.Options, to.Options) || !reflect.DeepEqual(from.GUI, to.GUI) {
+		return true
+	}
+
+	return false
+}
+
+func convertV3V4(cfg *Configuration) {
+	// In previous versions, rescan interval was common for each repository.
+	// From now, it can be set independently. We have to make sure, that after upgrade
+	// the individual rescan interval will be defined for every existing repository.
+	for i := range cfg.Repositories {
+		cfg.Repositories[i].RescanIntervalS = cfg.Options.Deprecated_RescanIntervalS
+	}
+
+	cfg.Options.Deprecated_RescanIntervalS = 0
+
+	// In previous versions, repositories held full node configurations.
+	// Since that's the only place where node configs were in V1, we still have
+	// to define the deprecated fields to be able to upgrade from V1 to V4.
+	for i, repo := range cfg.Repositories {
+
+		for j := range repo.Nodes {
+			rncfg := cfg.Repositories[i].Nodes[j]
+			rncfg.Deprecated_Name = ""
+			rncfg.Deprecated_Addresses = nil
+		}
+	}
+
+	cfg.Version = 4
+}
+
+func convertV2V3(cfg *Configuration) {
+	// In previous versions, compression was always on. When upgrading, enable
+	// compression on all existing new. New nodes will get compression on by
+	// default by the GUI.
+	for i := range cfg.Nodes {
+		cfg.Nodes[i].Compression = true
+	}
+
+	// The global discovery format and port number changed in v0.9. Having the
+	// default announce server but old port number is guaranteed to be legacy.
+	if cfg.Options.GlobalAnnServer == "announce.syncthing.net:22025" {
+		cfg.Options.GlobalAnnServer = "announce.syncthing.net:22026"
+	}
+
+	cfg.Version = 3
+}
+
+func convertV1V2(cfg *Configuration) {
+	// Collect the list of nodes.
+	// Replace node configs inside repositories with only a reference to the nide ID.
+	// Set all repositories to read only if the global read only flag is set.
+	var nodes = map[string]RepositoryNodeConfiguration{}
+	for i, repo := range cfg.Repositories {
+		cfg.Repositories[i].ReadOnly = cfg.Options.Deprecated_ReadOnly
+		for j, node := range repo.Nodes {
+			id := node.NodeID.String()
+			if _, ok := nodes[id]; !ok {
+				nodes[id] = node
+			}
+			cfg.Repositories[i].Nodes[j] = RepositoryNodeConfiguration{NodeID: node.NodeID}
+		}
+	}
+	cfg.Options.Deprecated_ReadOnly = false
+
+	// Set and sort the list of nodes.
+	for _, node := range nodes {
+		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
+			NodeID:    node.NodeID,
+			Name:      node.Deprecated_Name,
+			Addresses: node.Deprecated_Addresses,
+		})
+	}
+	sort.Sort(NodeConfigurationList(cfg.Nodes))
+
+	// GUI
+	cfg.GUI.Address = cfg.Options.Deprecated_GUIAddress
+	cfg.GUI.Enabled = cfg.Options.Deprecated_GUIEnabled
+	cfg.Options.Deprecated_GUIEnabled = false
+	cfg.Options.Deprecated_GUIAddress = ""
+
+	cfg.Version = 2
+}
+
+type NodeConfigurationList []NodeConfiguration
+
+func (l NodeConfigurationList) Less(a, b int) bool {
+	return l[a].NodeID.Compare(l[b].NodeID) == -1
+}
+func (l NodeConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l NodeConfigurationList) Len() int {
+	return len(l)
+}
+
+type RepositoryNodeConfigurationList []RepositoryNodeConfiguration
+
+func (l RepositoryNodeConfigurationList) Less(a, b int) bool {
+	return l[a].NodeID.Compare(l[b].NodeID) == -1
+}
+func (l RepositoryNodeConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l RepositoryNodeConfigurationList) Len() int {
+	return len(l)
+}
+
+func ensureNodePresent(nodes []RepositoryNodeConfiguration, myID protocol.NodeID) []RepositoryNodeConfiguration {
+	for _, node := range nodes {
+		if node.NodeID.Equals(myID) {
+			return nodes
+		}
+	}
+
+	nodes = append(nodes, RepositoryNodeConfiguration{
+		NodeID: myID,
+	})
+
+	return nodes
+}
+
+func ensureExistingNodes(nodes []RepositoryNodeConfiguration, existingNodes map[protocol.NodeID]bool) []RepositoryNodeConfiguration {
+	count := len(nodes)
+	i := 0
+loop:
+	for i < count {
+		if _, ok := existingNodes[nodes[i].NodeID]; !ok {
+			nodes[i] = nodes[count-1]
+			count--
+			continue loop
+		}
+		i++
+	}
+	return nodes[0:count]
+}
+
+func ensureNoDuplicates(nodes []RepositoryNodeConfiguration) []RepositoryNodeConfiguration {
+	count := len(nodes)
+	i := 0
+	seenNodes := make(map[protocol.NodeID]bool)
+loop:
+	for i < count {
+		id := nodes[i].NodeID
+		if _, ok := seenNodes[id]; ok {
+			nodes[i] = nodes[count-1]
+			count--
+			continue loop
+		}
+		seenNodes[id] = true
+		i++
+	}
+	return nodes[0:count]
+}

internal/config/config_test.go

@@ -0,0 +1,297 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package config
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+var node1, node2, node3, node4 protocol.NodeID
+
+func init() {
+	node1, _ = protocol.NodeIDFromString("AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ")
+	node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
+	node3, _ = protocol.NodeIDFromString("LGFPDIT-7SKNNJL-VJZA4FC-7QNCRKA-CE753K7-2BW5QDK-2FOZ7FR-FEP57QJ")
+	node4, _ = protocol.NodeIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
+}
+
+func TestDefaultValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{"0.0.0.0:22000"},
+		GlobalAnnServer:    "announce.syncthing.net:22026",
+		GlobalAnnEnabled:   true,
+		LocalAnnEnabled:    true,
+		LocalAnnPort:       21025,
+		LocalAnnMCAddr:     "[ff32::5222]:21026",
+		ParallelRequests:   16,
+		MaxSendKbps:        0,
+		MaxRecvKbps:        0,
+		ReconnectIntervalS: 60,
+		StartBrowser:       true,
+		UPnPEnabled:        true,
+		UPnPLease:          0,
+		UPnPRenewal:        30,
+		RestartOnWakeup:    true,
+	}
+
+	cfg := New("test", node1)
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestNodeConfig(t *testing.T) {
+	for i, ver := range []string{"v1", "v2", "v3", "v4"} {
+		cfg, err := Load("testdata/"+ver+".xml", node1)
+		if err != nil {
+			t.Error(err)
+		}
+
+		expectedRepos := []RepositoryConfiguration{
+			{
+				ID:              "test",
+				Directory:       "~/Sync",
+				Nodes:           []RepositoryNodeConfiguration{{NodeID: node1}, {NodeID: node4}},
+				ReadOnly:        true,
+				RescanIntervalS: 600,
+			},
+		}
+		expectedNodes := []NodeConfiguration{
+			{
+				NodeID:      node1,
+				Name:        "node one",
+				Addresses:   []string{"a"},
+				Compression: true,
+			},
+			{
+				NodeID:      node4,
+				Name:        "node two",
+				Addresses:   []string{"b"},
+				Compression: true,
+			},
+		}
+		expectedNodeIDs := []protocol.NodeID{node1, node4}
+
+		if cfg.Version != 4 {
+			t.Errorf("%d: Incorrect version %d != 3", i, cfg.Version)
+		}
+		if !reflect.DeepEqual(cfg.Repositories, expectedRepos) {
+			t.Errorf("%d: Incorrect Repositories\n  A: %#v\n  E: %#v", i, cfg.Repositories, expectedRepos)
+		}
+		if !reflect.DeepEqual(cfg.Nodes, expectedNodes) {
+			t.Errorf("%d: Incorrect Nodes\n  A: %#v\n  E: %#v", i, cfg.Nodes, expectedNodes)
+		}
+		if !reflect.DeepEqual(cfg.Repositories[0].NodeIDs(), expectedNodeIDs) {
+			t.Errorf("%d: Incorrect NodeIDs\n  A: %#v\n  E: %#v", i, cfg.Repositories[0].NodeIDs(), expectedNodeIDs)
+		}
+
+		if len(cfg.NodeMap()) != len(expectedNodes) {
+			t.Errorf("Unexpected number of NodeMap() entries")
+		}
+		if len(cfg.RepoMap()) != len(expectedRepos) {
+			t.Errorf("Unexpected number of RepoMap() entries")
+		}
+	}
+}
+
+func TestNoListenAddress(t *testing.T) {
+	cfg, err := Load("testdata/nolistenaddress.xml", node1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := []string{""}
+	if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
+		t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
+	}
+}
+
+func TestOverriddenValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":23000"},
+		GlobalAnnServer:    "syncthing.nym.se:22026",
+		GlobalAnnEnabled:   false,
+		LocalAnnEnabled:    false,
+		LocalAnnPort:       42123,
+		LocalAnnMCAddr:     "quux:3232",
+		ParallelRequests:   32,
+		MaxSendKbps:        1234,
+		MaxRecvKbps:        2341,
+		ReconnectIntervalS: 6000,
+		StartBrowser:       false,
+		UPnPEnabled:        false,
+		UPnPLease:          60,
+		UPnPRenewal:        15,
+		RestartOnWakeup:    false,
+	}
+
+	cfg, err := Load("testdata/overridenvalues.xml", node1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestNodeAddressesDynamic(t *testing.T) {
+	name, _ := os.Hostname()
+	expected := []NodeConfiguration{
+		{
+			NodeID:      node1,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
+		},
+		{
+			NodeID:      node2,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
+		},
+		{
+			NodeID:      node3,
+			Addresses:   []string{"dynamic"},
+			Compression: true,
+		},
+		{
+			NodeID:    node4,
+			Name:      name, // Set when auto created
+			Addresses: []string{"dynamic"},
+		},
+	}
+
+	cfg, err := Load("testdata/nodeaddressesdynamic.xml", node4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Nodes, expected) {
+		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
+	}
+}
+
+func TestNodeAddressesStatic(t *testing.T) {
+	name, _ := os.Hostname()
+	expected := []NodeConfiguration{
+		{
+			NodeID:    node1,
+			Addresses: []string{"192.0.2.1", "192.0.2.2"},
+		},
+		{
+			NodeID:    node2,
+			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
+		},
+		{
+			NodeID:    node3,
+			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
+		},
+		{
+			NodeID:    node4,
+			Name:      name, // Set when auto created
+			Addresses: []string{"dynamic"},
+		},
+	}
+
+	cfg, err := Load("testdata/nodeaddressesstatic.xml", node4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Nodes, expected) {
+		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
+	}
+}
+
+func TestVersioningConfig(t *testing.T) {
+	cfg, err := Load("testdata/versioningconfig.xml", node4)
+	if err != nil {
+		t.Error(err)
+	}
+
+	vc := cfg.Repositories[0].Versioning
+	if vc.Type != "simple" {
+		t.Errorf(`vc.Type %q != "simple"`, vc.Type)
+	}
+	if l := len(vc.Params); l != 2 {
+		t.Errorf("len(vc.Params) %d != 2", l)
+	}
+
+	expected := map[string]string{
+		"foo": "bar",
+		"baz": "quux",
+	}
+	if !reflect.DeepEqual(vc.Params, expected) {
+		t.Errorf("vc.Params differ;\n  E: %#v\n  A: %#v", expected, vc.Params)
+	}
+}
+
+func TestNewSaveLoad(t *testing.T) {
+	path := "testdata/temp.xml"
+	os.Remove(path)
+
+	exists := func(path string) bool {
+		_, err := os.Stat(path)
+		return err == nil
+	}
+
+	cfg := New(path, node1)
+
+	// To make the equality pass later
+	cfg.XMLName.Local = "configuration"
+
+	if exists(path) {
+		t.Error(path, "exists")
+	}
+
+	err := cfg.Save()
+	if err != nil {
+		t.Error(err)
+	}
+	if !exists(path) {
+		t.Error(path, "does not exist")
+	}
+
+	cfg2, err := Load(path, node1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg, cfg2) {
+		t.Errorf("Configs are not equal;\n  E:  %#v\n  A:  %#v", cfg, cfg2)
+	}
+
+	cfg.GUI.User = "test"
+	cfg.Save()
+
+	cfg2, err = Load(path, node1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if cfg2.GUI.User != "test" || !reflect.DeepEqual(cfg, cfg2) {
+		t.Errorf("Configs are not equal;\n  E:  %#v\n  A:  %#v", cfg, cfg2)
+	}
+
+	os.Remove(path)
+}
+
+func TestPrepare(t *testing.T) {
+	var cfg Configuration
+
+	if cfg.Repositories != nil || cfg.Nodes != nil || cfg.Options.ListenAddress != nil {
+		t.Error("Expected nil")
+	}
+
+	cfg.prepare(node1)
+
+	if cfg.Repositories == nil || cfg.Nodes == nil || cfg.Options.ListenAddress == nil {
+		t.Error("Unexpected nil")
+	}
+}

internal/config/testdata/nodeaddressesdynamic.xml

@@ -0,0 +1,10 @@
+<configuration version="2">
+    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
+        <address></address>
+    </node>
+    <node id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
+    </node>
+    <node id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
+        <address>dynamic</address>
+    </node>
+</configuration>

internal/config/testdata/nodeaddressesstatic.xml

@@ -0,0 +1,14 @@
+<configuration version="3">
+    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
+        <address>192.0.2.1</address>
+        <address>192.0.2.2</address>
+    </node>
+    <node id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
+        <address>192.0.2.3:6070</address>
+        <address>[2001:db8::42]:4242</address>
+    </node>
+    <node id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
+        <address>[2001:db8::44]:4444</address>
+        <address>192.0.2.4:6090</address>
+    </node>
+</configuration>

internal/config/testdata/nolistenaddress.xml

@@ -0,0 +1,5 @@
+<configuration version="1">
+    <options>
+        <listenAddress></listenAddress>
+    </options>
+</configuration>

internal/config/testdata/overridenvalues.xml

@@ -0,0 +1,20 @@
+<configuration version="2">
+    <options>
+       <listenAddress>:23000</listenAddress>
+        <allowDelete>false</allowDelete>
+        <globalAnnounceServer>syncthing.nym.se:22026</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <localAnnouncePort>42123</localAnnouncePort>
+        <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
+        <parallelRequests>32</parallelRequests>
+        <maxSendKbps>1234</maxSendKbps>
+        <maxRecvKbps>2341</maxRecvKbps>
+        <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <startBrowser>false</startBrowser>
+        <upnpEnabled>false</upnpEnabled>
+        <upnpLeaseMinutes>60</upnpLeaseMinutes>
+        <upnpRenewalMinutes>15</upnpRenewalMinutes>
+        <restartOnWakeup>false</restartOnWakeup>
+    </options>
+</configuration>

internal/config/testdata/v1.xml

@@ -0,0 +1,20 @@
+<configuration version="1">
+    <repository id="test" directory="~/Sync">
+        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
+            <address>a</address>
+        </node>
+        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
+            <address>b</address>
+        </node>
+        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
+            <address>a</address>
+        </node>
+        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
+            <address>b</address>
+        </node>
+    </repository>
+    <options>
+        <readOnly>true</readOnly>
+        <rescanIntervalS>600</rescanIntervalS>
+    </options>
+</configuration>

internal/config/testdata/v2.xml

@@ -0,0 +1,19 @@
+<configuration version="2">
+    <repository id="test" directory="~/Sync" ro="true">
+        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
+        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
+        <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>
+        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
+        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
+        <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>
+    </repository>
+    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
+        <address>a</address>
+    </node>
+    <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
+        <address>b</address>
+    </node>
+    <options>
+        <rescanIntervalS>600</rescanIntervalS>
+    </options>
+</configuration>

internal/config/testdata/v3.xml

@@ -0,0 +1,15 @@
+<configuration version="3">
+    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false">
+        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" compression="false"></node>
+        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" compression="false"></node>
+    </repository>
+    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </node>
+    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </node>
+    <options>
+        <rescanIntervalS>600</rescanIntervalS>
+    </options>
+</configuration>

internal/config/testdata/v4.xml

@@ -0,0 +1,12 @@
+<configuration version="4">
+    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></node>
+        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></node>
+    </repository>
+    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </node>
+    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </node>
+</configuration>

internal/config/testdata/versioningconfig.xml

@@ -0,0 +1,8 @@
+<configuration version="2">
+    <repository id="test" directory="~/Sync" ro="true">
+        <versioning type="simple">
+            <param key="foo" val="bar"/>
+            <param key="baz" val="quux"/>
+        </versioning>
+    </repository>
+</configuration>

internal/discover/PROTOCOL.md

@@ -0,0 +1,138 @@
+Node Discovery Protocol v2
+==========================
+
+Mode of Operation
+-----------------
+
+There are two distinct modes: "local discovery", performed on a LAN
+segment (broadcast domain) and "global discovery" performed over the
+Internet in general with the support of a well known server.
+
+Local discovery does not use Query packets. Instead Announcement packets
+are sent periodically and each participating node keeps a table of the
+announcements it has seen. On multihomed hosts the announcement packets
+should be sent on each interface that syncthing will accept connections.
+
+It is recommended that local discovery Announcement packets are sent on
+a 30 to 60 second interval, possibly with forced transmissions when a
+previously unknown node is discovered.
+
+Global discovery is made possible by periodically updating a global server
+using Announcement packets indentical to those transmitted for local
+discovery. The node performing discovery will transmit a Query packet to
+the global server and expect an Announcement packet in response. In case
+the global server has no knowledge of the queried node ID, there will be
+no response. A timeout is to be used to determine lookup failure.
+
+There is no message to unregister from the global server; instead
+registrations are forgotten after 60 minutes. It is recommended to
+send Announcement packets to the global server on a 30 minute interval.
+
+Packet Formats
+--------------
+
+The Announcement packet has the following structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                      Magic (0x9D79BC39)                       |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                        Node Structure                         \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Number of Extra Nodes                     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                 Zero or more Node Structures                  \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    Node Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of ID                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     ID (variable length)                      \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                      Number of Addresses                      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                Zero or more Address Structures                \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    Address Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of IP                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     IP (variable length)                      \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |             Port              |            0x0000             |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+This is the XDR encoding of:
+
+    struct Announcement {
+        unsigned int Magic;
+        Node This;
+        Node Extra<>;
+    }
+
+    struct Node {
+        string ID<>;
+        Address Addresses<>;
+    }
+
+    struct Address {
+        opaque IP<>;
+        unsigned short Port;
+    }
+
+The first Node structure contains information about the sending node.
+The following zero or more Extra nodes contain information about other
+nodes known to the sending node.
+
+In the Address structure, the IP field can be of three differnt kinds;
+
+ - A zero length indicates that the IP address should be taken from the
+   source address of the announcement packet, be it IPv4 or IPv6. The
+   source address must be a valid unicast address. This is only valid
+   in the first node structure, not in the list of extras.
+
+ - A four byte length indicates that the address is an IPv4 unicast
+   address.
+
+ - A sixteen byte length indicates that the address is an IPv6 unicast
+   address.
+
+The Query packet has the following structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                   Magic Number (0x2CA856F5)                   |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                       Length of Node ID                       |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                   Node ID (variable length)                   \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+This is the XDR encoding of:
+
+    struct Announcement {
+        unsigned int MagicNumber;
+        string NodeID<>;
+    }

internal/discover/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package discover
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/discover/discover.go

@@ -0,0 +1,501 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package discover
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/beacon"
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+type Discoverer struct {
+	myID             protocol.NodeID
+	listenAddrs      []string
+	localBcastIntv   time.Duration
+	globalBcastIntv  time.Duration
+	errorRetryIntv   time.Duration
+	cacheLifetime    time.Duration
+	broadcastBeacon  beacon.Interface
+	multicastBeacon  beacon.Interface
+	registry         map[protocol.NodeID][]cacheEntry
+	registryLock     sync.RWMutex
+	extServer        string
+	extPort          uint16
+	localBcastTick   <-chan time.Time
+	stopGlobal       chan struct{}
+	globalWG         sync.WaitGroup
+	forcedBcastTick  chan time.Time
+	extAnnounceOK    bool
+	extAnnounceOKmut sync.Mutex
+}
+
+type cacheEntry struct {
+	addr string
+	seen time.Time
+}
+
+var (
+	ErrIncorrectMagic = errors.New("incorrect magic number")
+)
+
+func NewDiscoverer(id protocol.NodeID, addresses []string) *Discoverer {
+	return &Discoverer{
+		myID:            id,
+		listenAddrs:     addresses,
+		localBcastIntv:  30 * time.Second,
+		globalBcastIntv: 1800 * time.Second,
+		errorRetryIntv:  60 * time.Second,
+		cacheLifetime:   5 * time.Minute,
+		registry:        make(map[protocol.NodeID][]cacheEntry),
+	}
+}
+
+func (d *Discoverer) StartLocal(localPort int, localMCAddr string) {
+	if localPort > 0 {
+		bb, err := beacon.NewBroadcast(localPort)
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			l.Infoln("Local discovery over IPv4 unavailable")
+		} else {
+			d.broadcastBeacon = bb
+			go d.recvAnnouncements(bb)
+		}
+	}
+
+	if len(localMCAddr) > 0 {
+		mb, err := beacon.NewMulticast(localMCAddr)
+		if err != nil {
+			if debug {
+				l.Debugln(err)
+			}
+			l.Infoln("Local discovery over IPv6 unavailable")
+		} else {
+			d.multicastBeacon = mb
+			go d.recvAnnouncements(mb)
+		}
+	}
+
+	if d.broadcastBeacon == nil && d.multicastBeacon == nil {
+		l.Warnln("Local discovery unavailable")
+	} else {
+		d.localBcastTick = time.Tick(d.localBcastIntv)
+		d.forcedBcastTick = make(chan time.Time)
+		go d.sendLocalAnnouncements()
+	}
+}
+
+func (d *Discoverer) StartGlobal(server string, extPort uint16) {
+	// Wait for any previous announcer to stop before starting a new one.
+	d.globalWG.Wait()
+	d.extServer = server
+	d.extPort = extPort
+	d.stopGlobal = make(chan struct{})
+	d.globalWG.Add(1)
+	go d.sendExternalAnnouncements()
+}
+
+func (d *Discoverer) StopGlobal() {
+	if d.stopGlobal != nil {
+		close(d.stopGlobal)
+		d.globalWG.Wait()
+	}
+}
+
+func (d *Discoverer) ExtAnnounceOK() bool {
+	d.extAnnounceOKmut.Lock()
+	defer d.extAnnounceOKmut.Unlock()
+	return d.extAnnounceOK
+}
+
+func (d *Discoverer) Lookup(node protocol.NodeID) []string {
+	d.registryLock.Lock()
+	cached := d.filterCached(d.registry[node])
+	d.registryLock.Unlock()
+
+	if len(cached) > 0 {
+		addrs := make([]string, len(cached))
+		for i := range cached {
+			addrs[i] = cached[i].addr
+		}
+		return addrs
+	} else if len(d.extServer) != 0 {
+		addrs := d.externalLookup(node)
+		cached = make([]cacheEntry, len(addrs))
+		for i := range addrs {
+			cached[i] = cacheEntry{
+				addr: addrs[i],
+				seen: time.Now(),
+			}
+		}
+
+		d.registryLock.Lock()
+		d.registry[node] = cached
+		d.registryLock.Unlock()
+	}
+	return nil
+}
+
+func (d *Discoverer) Hint(node string, addrs []string) {
+	resAddrs := resolveAddrs(addrs)
+	var id protocol.NodeID
+	id.UnmarshalText([]byte(node))
+	d.registerNode(nil, Node{
+		Addresses: resAddrs,
+		ID:        id[:],
+	})
+}
+
+func (d *Discoverer) All() map[protocol.NodeID][]cacheEntry {
+	d.registryLock.RLock()
+	nodes := make(map[protocol.NodeID][]cacheEntry, len(d.registry))
+	for node, addrs := range d.registry {
+		addrsCopy := make([]cacheEntry, len(addrs))
+		copy(addrsCopy, addrs)
+		nodes[node] = addrsCopy
+	}
+	d.registryLock.RUnlock()
+	return nodes
+}
+
+func (d *Discoverer) announcementPkt() []byte {
+	var addrs []Address
+	for _, astr := range d.listenAddrs {
+		addr, err := net.ResolveTCPAddr("tcp", astr)
+		if err != nil {
+			l.Warnln("%v: not announcing %s", err, astr)
+			continue
+		} else if debug {
+			l.Debugf("discover: announcing %s: %#v", astr, addr)
+		}
+		if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+			addrs = append(addrs, Address{Port: uint16(addr.Port)})
+		} else if bs := addr.IP.To4(); bs != nil {
+			addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
+		} else if bs := addr.IP.To16(); bs != nil {
+			addrs = append(addrs, Address{IP: bs, Port: uint16(addr.Port)})
+		}
+	}
+	var pkt = Announce{
+		Magic: AnnouncementMagic,
+		This:  Node{d.myID[:], addrs},
+	}
+	return pkt.MarshalXDR()
+}
+
+func (d *Discoverer) sendLocalAnnouncements() {
+	var addrs = resolveAddrs(d.listenAddrs)
+
+	var pkt = Announce{
+		Magic: AnnouncementMagic,
+		This:  Node{d.myID[:], addrs},
+	}
+	msg := pkt.MarshalXDR()
+
+	for {
+		if d.multicastBeacon != nil {
+			d.multicastBeacon.Send(msg)
+		}
+		if d.broadcastBeacon != nil {
+			d.broadcastBeacon.Send(msg)
+		}
+
+		select {
+		case <-d.localBcastTick:
+		case <-d.forcedBcastTick:
+		}
+	}
+}
+
+func (d *Discoverer) sendExternalAnnouncements() {
+	defer d.globalWG.Done()
+
+	remote, err := net.ResolveUDPAddr("udp", d.extServer)
+	for err != nil {
+		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
+		time.Sleep(d.errorRetryIntv)
+		remote, err = net.ResolveUDPAddr("udp", d.extServer)
+	}
+
+	conn, err := net.ListenUDP("udp", nil)
+	for err != nil {
+		l.Warnf("Global discovery: %v; trying again in %v", err, d.errorRetryIntv)
+		time.Sleep(d.errorRetryIntv)
+		conn, err = net.ListenUDP("udp", nil)
+	}
+
+	var buf []byte
+	if d.extPort != 0 {
+		var pkt = Announce{
+			Magic: AnnouncementMagic,
+			This:  Node{d.myID[:], []Address{{Port: d.extPort}}},
+		}
+		buf = pkt.MarshalXDR()
+	} else {
+		buf = d.announcementPkt()
+	}
+
+	var bcastTick = time.Tick(d.globalBcastIntv)
+	var errTick <-chan time.Time
+
+	sendOneAnnouncement := func() {
+		var ok bool
+
+		if debug {
+			l.Debugf("discover: send announcement -> %v\n%s", remote, hex.Dump(buf))
+		}
+
+		_, err := conn.WriteTo(buf, remote)
+		if err != nil {
+			if debug {
+				l.Debugln("discover: warning:", err)
+			}
+			ok = false
+		} else {
+			// Verify that the announce server responds positively for our node ID
+
+			time.Sleep(1 * time.Second)
+			res := d.externalLookup(d.myID)
+			if debug {
+				l.Debugln("discover: external lookup check:", res)
+			}
+			ok = len(res) > 0
+		}
+
+		d.extAnnounceOKmut.Lock()
+		d.extAnnounceOK = ok
+		d.extAnnounceOKmut.Unlock()
+
+		if ok {
+			errTick = nil
+		} else if errTick != nil {
+			errTick = time.Tick(d.errorRetryIntv)
+		}
+	}
+
+	// Announce once, immediately
+	sendOneAnnouncement()
+
+loop:
+	for {
+		select {
+		case <-d.stopGlobal:
+			break loop
+
+		case <-errTick:
+			sendOneAnnouncement()
+
+		case <-bcastTick:
+			sendOneAnnouncement()
+		}
+	}
+
+	if debug {
+		l.Debugln("discover: stopping global")
+	}
+}
+
+func (d *Discoverer) recvAnnouncements(b beacon.Interface) {
+	for {
+		buf, addr := b.Recv()
+
+		if debug {
+			l.Debugf("discover: read announcement from %s:\n%s", addr, hex.Dump(buf))
+		}
+
+		var pkt Announce
+		err := pkt.UnmarshalXDR(buf)
+		if err != nil && err != io.EOF {
+			continue
+		}
+
+		var newNode bool
+		if bytes.Compare(pkt.This.ID, d.myID[:]) != 0 {
+			newNode = d.registerNode(addr, pkt.This)
+		}
+
+		if newNode {
+			select {
+			case d.forcedBcastTick <- time.Now():
+			}
+		}
+	}
+}
+
+func (d *Discoverer) registerNode(addr net.Addr, node Node) bool {
+	var id protocol.NodeID
+	copy(id[:], node.ID)
+
+	d.registryLock.RLock()
+	current := d.filterCached(d.registry[id])
+	d.registryLock.RUnlock()
+
+	orig := current
+
+	for _, a := range node.Addresses {
+		var nodeAddr string
+		if len(a.IP) > 0 {
+			nodeAddr = net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
+		} else if addr != nil {
+			ua := addr.(*net.UDPAddr)
+			ua.Port = int(a.Port)
+			nodeAddr = ua.String()
+		}
+		for i := range current {
+			if current[i].addr == nodeAddr {
+				current[i].seen = time.Now()
+				goto done
+			}
+		}
+		current = append(current, cacheEntry{
+			addr: nodeAddr,
+			seen: time.Now(),
+		})
+	done:
+	}
+
+	if debug {
+		l.Debugf("discover: register: %v -> %v", id, current)
+	}
+
+	d.registryLock.Lock()
+	d.registry[id] = current
+	d.registryLock.Unlock()
+
+	if len(current) > len(orig) {
+		addrs := make([]string, len(current))
+		for i := range current {
+			addrs[i] = current[i].addr
+		}
+		events.Default.Log(events.NodeDiscovered, map[string]interface{}{
+			"node":  id.String(),
+			"addrs": addrs,
+		})
+	}
+
+	return len(current) > len(orig)
+}
+
+func (d *Discoverer) externalLookup(node protocol.NodeID) []string {
+	extIP, err := net.ResolveUDPAddr("udp", d.extServer)
+	if err != nil {
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
+		return nil
+	}
+
+	conn, err := net.DialUDP("udp", nil, extIP)
+	if err != nil {
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
+		return nil
+	}
+	defer conn.Close()
+
+	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
+	if err != nil {
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
+		return nil
+	}
+
+	buf := Query{QueryMagic, node[:]}.MarshalXDR()
+	_, err = conn.Write(buf)
+	if err != nil {
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
+		return nil
+	}
+
+	buf = make([]byte, 2048)
+	n, err := conn.Read(buf)
+	if err != nil {
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			// Expected if the server doesn't know about requested node ID
+			return nil
+		}
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
+		return nil
+	}
+
+	if debug {
+		l.Debugf("discover: read external:\n%s", hex.Dump(buf[:n]))
+	}
+
+	var pkt Announce
+	err = pkt.UnmarshalXDR(buf[:n])
+	if err != nil && err != io.EOF {
+		if debug {
+			l.Debugln("discover:", err)
+		}
+		return nil
+	}
+
+	var addrs []string
+	for _, a := range pkt.This.Addresses {
+		nodeAddr := net.JoinHostPort(net.IP(a.IP).String(), strconv.Itoa(int(a.Port)))
+		addrs = append(addrs, nodeAddr)
+	}
+	return addrs
+}
+
+func (d *Discoverer) filterCached(c []cacheEntry) []cacheEntry {
+	for i := 0; i < len(c); {
+		if ago := time.Since(c[i].seen); ago > d.cacheLifetime {
+			if debug {
+				l.Debugf("removing cached address %s: seen %v ago", c[i].addr, ago)
+			}
+			c[i] = c[len(c)-1]
+			c = c[:len(c)-1]
+		} else {
+			i++
+		}
+	}
+	return c
+}
+
+func addrToAddr(addr *net.TCPAddr) Address {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return Address{Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To4(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To16(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	}
+	return Address{}
+}
+
+func resolveAddrs(addrs []string) []Address {
+	var raddrs []Address
+	for _, addrStr := range addrs {
+		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr.IP) > 0 {
+			raddrs = append(raddrs, addr)
+		} else {
+			raddrs = append(raddrs, Address{Port: addr.Port})
+		}
+	}
+	return raddrs
+}

internal/discover/discover_test.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package discover_test
+
+// Empty test file to generate 0% coverage rather than no coverage

internal/discover/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package discover implements the node discovery protocol.
+package discover

internal/discover/packets.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package discover
+
+const (
+	AnnouncementMagic = 0x9D79BC39
+	QueryMagic        = 0x2CA856F5
+)
+
+type Query struct {
+	Magic  uint32
+	NodeID []byte // max:32
+}
+
+type Announce struct {
+	Magic uint32
+	This  Node
+	Extra []Node // max:16
+}
+
+type Node struct {
+	ID        []byte    // max:32
+	Addresses []Address // max:16
+}
+
+type Address struct {
+	IP   []byte // max:16
+	Port uint16
+}

internal/discover/packets_xdr.go

@@ -0,0 +1,317 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package discover
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+Query Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Magic                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Length of Node ID                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                   Node ID (variable length)                   \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Query {
+	unsigned int Magic;
+	opaque NodeID<32>;
+}
+
+*/
+
+func (o Query) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Query) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Query) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Query) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.Magic)
+	if len(o.NodeID) > 32 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.NodeID)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Query) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Query) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Query) decodeXDR(xr *xdr.Reader) error {
+	o.Magic = xr.ReadUint32()
+	o.NodeID = xr.ReadBytesMax(32)
+	return xr.Error()
+}
+
+/*
+
+Announce Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Magic                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Node                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Number of Extra                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Zero or more Node Structures                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Announce {
+	unsigned int Magic;
+	Node This;
+	Node Extra<16>;
+}
+
+*/
+
+func (o Announce) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Announce) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Announce) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Announce) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.Magic)
+	_, err := o.This.encodeXDR(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
+	if len(o.Extra) > 16 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Extra)))
+	for i := range o.Extra {
+		_, err := o.Extra[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Announce) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Announce) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Announce) decodeXDR(xr *xdr.Reader) error {
+	o.Magic = xr.ReadUint32()
+	(&o.This).decodeXDR(xr)
+	_ExtraSize := int(xr.ReadUint32())
+	if _ExtraSize > 16 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Extra = make([]Node, _ExtraSize)
+	for i := range o.Extra {
+		(&o.Extra[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Node Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of ID                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     ID (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Number of Addresses                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Address Structures                \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Node {
+	opaque ID<32>;
+	Address Addresses<16>;
+}
+
+*/
+
+func (o Node) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Node) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Node) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 32 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.ID)
+	if len(o.Addresses) > 16 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Addresses)))
+	for i := range o.Addresses {
+		_, err := o.Addresses[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Node) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadBytesMax(32)
+	_AddressesSize := int(xr.ReadUint32())
+	if _AddressesSize > 16 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Addresses = make([]Address, _AddressesSize)
+	for i := range o.Addresses {
+		(&o.Addresses[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Address Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of IP                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     IP (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |             Port              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Address {
+	opaque IP<16>;
+	unsigned int Port;
+}
+
+*/
+
+func (o Address) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Address) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Address) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Address) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.IP) > 16 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.IP)
+	xw.WriteUint16(o.Port)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Address) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Address) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Address) decodeXDR(xr *xdr.Reader) error {
+	o.IP = xr.ReadBytesMax(16)
+	o.Port = xr.ReadUint16()
+	return xr.Error()
+}

internal/events/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package events
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all"
+	dl    = logger.DefaultLogger
+)

internal/events/events.go

@@ -0,0 +1,237 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package events provides event subscription and polling functionality.
+package events
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+type EventType uint64
+
+const (
+	Ping = 1 << iota
+	Starting
+	StartupComplete
+	NodeDiscovered
+	NodeConnected
+	NodeDisconnected
+	NodeRejected
+	LocalIndexUpdated
+	RemoteIndexUpdated
+	ItemStarted
+	StateChanged
+	RepoRejected
+	ConfigSaved
+
+	AllEvents = ^EventType(0)
+)
+
+func (t EventType) String() string {
+	switch t {
+	case Ping:
+		return "Ping"
+	case Starting:
+		return "Starting"
+	case StartupComplete:
+		return "StartupComplete"
+	case NodeDiscovered:
+		return "NodeDiscovered"
+	case NodeConnected:
+		return "NodeConnected"
+	case NodeDisconnected:
+		return "NodeDisconnected"
+	case NodeRejected:
+		return "NodeRejected"
+	case LocalIndexUpdated:
+		return "LocalIndexUpdated"
+	case RemoteIndexUpdated:
+		return "RemoteIndexUpdated"
+	case ItemStarted:
+		return "ItemStarted"
+	case StateChanged:
+		return "StateChanged"
+	case RepoRejected:
+		return "RepoRejected"
+	case ConfigSaved:
+		return "ConfigSaved"
+	default:
+		return "Unknown"
+	}
+}
+
+func (t EventType) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+const BufferSize = 64
+
+type Logger struct {
+	subs   map[int]*Subscription
+	nextId int
+	mutex  sync.Mutex
+}
+
+type Event struct {
+	ID   int         `json:"id"`
+	Time time.Time   `json:"time"`
+	Type EventType   `json:"type"`
+	Data interface{} `json:"data"`
+}
+
+type Subscription struct {
+	mask   EventType
+	id     int
+	events chan Event
+	mutex  sync.Mutex
+}
+
+var Default = NewLogger()
+
+var (
+	ErrTimeout = errors.New("timeout")
+	ErrClosed  = errors.New("closed")
+)
+
+func NewLogger() *Logger {
+	return &Logger{
+		subs: make(map[int]*Subscription),
+	}
+}
+
+func (l *Logger) Log(t EventType, data interface{}) {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("log", l.nextId, t.String(), data)
+	}
+	e := Event{
+		ID:   l.nextId,
+		Time: time.Now(),
+		Type: t,
+		Data: data,
+	}
+	l.nextId++
+	for _, s := range l.subs {
+		if s.mask&t != 0 {
+			select {
+			case s.events <- e:
+			default:
+				//log.Println("Dropping event:", e)
+			}
+		}
+	}
+	l.mutex.Unlock()
+}
+
+func (l *Logger) Subscribe(mask EventType) *Subscription {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("subscribe", mask)
+	}
+	s := &Subscription{
+		mask:   mask,
+		id:     l.nextId,
+		events: make(chan Event, BufferSize),
+	}
+	l.nextId++
+	l.subs[s.id] = s
+	l.mutex.Unlock()
+	return s
+}
+
+func (l *Logger) Unsubscribe(s *Subscription) {
+	l.mutex.Lock()
+	if debug {
+		dl.Debugln("unsubsribe")
+	}
+	delete(l.subs, s.id)
+	close(s.events)
+	l.mutex.Unlock()
+}
+
+func (s *Subscription) Poll(timeout time.Duration) (Event, error) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	if debug {
+		dl.Debugln("poll", timeout)
+	}
+
+	to := time.After(timeout)
+	select {
+	case e, ok := <-s.events:
+		if !ok {
+			return e, ErrClosed
+		}
+		return e, nil
+	case <-to:
+		return Event{}, ErrTimeout
+	}
+}
+
+type BufferedSubscription struct {
+	sub  *Subscription
+	buf  []Event
+	next int
+	cur  int
+	mut  sync.Mutex
+	cond *sync.Cond
+}
+
+func NewBufferedSubscription(s *Subscription, size int) *BufferedSubscription {
+	bs := &BufferedSubscription{
+		sub: s,
+		buf: make([]Event, size),
+	}
+	bs.cond = sync.NewCond(&bs.mut)
+	go bs.pollingLoop()
+	return bs
+}
+
+func (s *BufferedSubscription) pollingLoop() {
+	for {
+		ev, err := s.sub.Poll(60 * time.Second)
+		if err == ErrTimeout {
+			continue
+		}
+		if err == ErrClosed {
+			return
+		}
+		if err != nil {
+			panic("unexpected error: " + err.Error())
+		}
+
+		s.mut.Lock()
+		s.buf[s.next] = ev
+		s.next = (s.next + 1) % len(s.buf)
+		s.cur = ev.ID
+		s.cond.Broadcast()
+		s.mut.Unlock()
+	}
+}
+
+func (s *BufferedSubscription) Since(id int, into []Event) []Event {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+
+	for id >= s.cur {
+		s.cond.Wait()
+	}
+
+	for i := s.next; i < len(s.buf); i++ {
+		if s.buf[i].ID > id {
+			into = append(into, s.buf[i])
+		}
+	}
+	for i := 0; i < s.next; i++ {
+		if s.buf[i].ID > id {
+			into = append(into, s.buf[i])
+		}
+	}
+
+	return into
+}

internal/events/events_test.go

@@ -0,0 +1,178 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package events_test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/events"
+)
+
+var timeout = 100 * time.Millisecond
+
+func TestNewLogger(t *testing.T) {
+	l := events.NewLogger()
+	if l == nil {
+		t.Fatal("Unexpected nil Logger")
+	}
+}
+
+func TestSubscriber(t *testing.T) {
+	l := events.NewLogger()
+	s := l.Subscribe(0)
+	if s == nil {
+		t.Fatal("Unexpected nil Subscription")
+	}
+}
+
+func TestTimeout(t *testing.T) {
+	l := events.NewLogger()
+	s := l.Subscribe(0)
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestEventBeforeSubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	l.Log(events.NodeConnected, "foo")
+	s := l.Subscribe(0)
+
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestEventAfterSubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.NodeConnected, "foo")
+
+	ev, err := s.Poll(timeout)
+
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Type != events.NodeConnected {
+		t.Error("Incorrect event type", ev.Type)
+	}
+	switch v := ev.Data.(type) {
+	case string:
+		if v != "foo" {
+			t.Error("Incorrect Data string", v)
+		}
+	default:
+		t.Errorf("Incorrect Data type %#v", v)
+	}
+}
+
+func TestEventAfterSubscribeIgnoreMask(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.NodeDisconnected)
+	l.Log(events.NodeConnected, "foo")
+
+	_, err := s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected non-Timeout error:", err)
+	}
+}
+
+func TestBufferOverflow(t *testing.T) {
+	l := events.NewLogger()
+
+	_ = l.Subscribe(events.AllEvents)
+
+	t0 := time.Now()
+	for i := 0; i < events.BufferSize*2; i++ {
+		l.Log(events.NodeConnected, "foo")
+	}
+	if time.Since(t0) > timeout {
+		t.Fatalf("Logging took too long")
+	}
+}
+
+func TestUnsubscribe(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.NodeConnected, "foo")
+
+	_, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	l.Unsubscribe(s)
+	l.Log(events.NodeConnected, "foo")
+
+	_, err = s.Poll(timeout)
+	if err != events.ErrClosed {
+		t.Fatal("Unexpected non-Closed error:", err)
+	}
+}
+
+func TestIDs(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	l.Log(events.NodeConnected, "foo")
+	l.Log(events.NodeConnected, "bar")
+
+	ev, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Data.(string) != "foo" {
+		t.Fatal("Incorrect event:", ev)
+	}
+	id := ev.ID
+
+	ev, err = s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.Data.(string) != "bar" {
+		t.Fatal("Incorrect event:", ev)
+	}
+	if !(ev.ID > id) {
+		t.Fatalf("ID not incremented (%d !> %d)", ev.ID, id)
+	}
+}
+
+func TestBufferedSub(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	bs := events.NewBufferedSubscription(s, 10*events.BufferSize)
+
+	go func() {
+		for i := 0; i < 10*events.BufferSize; i++ {
+			l.Log(events.NodeConnected, fmt.Sprintf("event-%d", i))
+			if i%30 == 0 {
+				// Give the buffer routine time to pick up the events
+				time.Sleep(20 * time.Millisecond)
+			}
+		}
+	}()
+
+	recv := 0
+	for recv < 10*events.BufferSize {
+		evs := bs.Since(recv, nil)
+		for _, ev := range evs {
+			if ev.ID != recv+1 {
+				t.Fatalf("Incorrect ID; %d != %d", ev.ID, recv+1)
+			}
+			recv = ev.ID
+		}
+	}
+
+}

internal/files/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package files
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/files/filenames_darwin.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package files
+
+import "code.google.com/p/go.text/unicode/norm"
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(s)
+}
+
+func nativeFilename(s string) string {
+	return norm.NFD.String(s)
+}

internal/files/filenames_unix.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !windows,!darwin
+
+package files
+
+import "code.google.com/p/go.text/unicode/norm"
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(s)
+}
+
+func nativeFilename(s string) string {
+	return s
+}

internal/files/filenames_windows.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package files
+
+import (
+	"path/filepath"
+
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+func normalizedFilename(s string) string {
+	return norm.NFC.String(filepath.ToSlash(s))
+}
+
+func nativeFilename(s string) string {
+	return filepath.FromSlash(s)
+}

internal/files/leveldb.go

@@ -0,0 +1,763 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package files
+
+import (
+	"bytes"
+	"runtime"
+	"sort"
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/lamport"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	clockTick uint64
+	clockMut  sync.Mutex
+)
+
+func clock(v uint64) uint64 {
+	clockMut.Lock()
+	defer clockMut.Unlock()
+	if v > clockTick {
+		clockTick = v + 1
+	} else {
+		clockTick++
+	}
+	return clockTick
+}
+
+const (
+	keyTypeNode = iota
+	keyTypeGlobal
+)
+
+type fileVersion struct {
+	version uint64
+	node    []byte
+}
+
+type versionList struct {
+	versions []fileVersion
+}
+
+type fileList []protocol.FileInfo
+
+func (l fileList) Len() int {
+	return len(l)
+}
+
+func (l fileList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l fileList) Less(a, b int) bool {
+	return l[a].Name < l[b].Name
+}
+
+type dbReader interface {
+	Get([]byte, *opt.ReadOptions) ([]byte, error)
+}
+
+type dbWriter interface {
+	Put([]byte, []byte)
+	Delete([]byte)
+}
+
+/*
+
+keyTypeNode (1 byte)
+    repository (64 bytes)
+        node (32 bytes)
+            name (variable size)
+		|
+		scanner.File
+
+keyTypeGlobal (1 byte)
+	repository (64 bytes)
+		name (variable size)
+			|
+			[]fileVersion (sorted)
+
+*/
+
+func nodeKey(repo, node, file []byte) []byte {
+	k := make([]byte, 1+64+32+len(file))
+	k[0] = keyTypeNode
+	copy(k[1:], []byte(repo))
+	copy(k[1+64:], node[:])
+	copy(k[1+64+32:], []byte(file))
+	return k
+}
+
+func globalKey(repo, file []byte) []byte {
+	k := make([]byte, 1+64+len(file))
+	k[0] = keyTypeGlobal
+	copy(k[1:], []byte(repo))
+	copy(k[1+64:], []byte(file))
+	return k
+}
+
+func nodeKeyName(key []byte) []byte {
+	return key[1+64+32:]
+}
+func nodeKeyRepo(key []byte) []byte {
+	repo := key[1 : 1+64]
+	izero := bytes.IndexByte(repo, 0)
+	return repo[:izero]
+}
+func nodeKeyNode(key []byte) []byte {
+	return key[1+64 : 1+64+32]
+}
+
+func globalKeyName(key []byte) []byte {
+	return key[1+64:]
+}
+
+func globalKeyRepo(key []byte) []byte {
+	repo := key[1 : 1+64]
+	izero := bytes.IndexByte(repo, 0)
+	return repo[:izero]
+}
+
+type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64
+
+type fileIterator func(f protocol.FileIntf) bool
+
+func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
+	runtime.GC()
+
+	sort.Sort(fileList(fs)) // sort list on name, same as on disk
+
+	start := nodeKey(repo, node, nil)                            // before all repo/node files
+	limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
+
+	batch := new(leveldb.Batch)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	moreDb := dbi.Next()
+	fsi := 0
+	var maxLocalVer uint64
+
+	for {
+		var newName, oldName []byte
+		moreFs := fsi < len(fs)
+
+		if !moreDb && !moreFs {
+			break
+		}
+
+		if !moreFs && deleteFn == nil {
+			// We don't have any more updated files to process and deletion
+			// has not been requested, so we can exit early
+			break
+		}
+
+		if moreFs {
+			newName = []byte(fs[fsi].Name)
+		}
+
+		if moreDb {
+			oldName = nodeKeyName(dbi.Key())
+		}
+
+		cmp := bytes.Compare(newName, oldName)
+
+		if debug {
+			l.Debugf("generic replace; repo=%q node=%v moreFs=%v moreDb=%v cmp=%d newName=%q oldName=%q", repo, protocol.NodeIDFromBytes(node), moreFs, moreDb, cmp, newName, oldName)
+		}
+
+		switch {
+		case moreFs && (!moreDb || cmp == -1):
+			// Disk is missing this file. Insert it.
+			if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
+				maxLocalVer = lv
+			}
+			if fs[fsi].IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, newName)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+			}
+			fsi++
+
+		case moreFs && moreDb && cmp == 0:
+			// File exists on both sides - compare versions. We might get an
+			// update with the same version and different flags if a node has
+			// marked a file as invalid, so handle that too.
+			var ef protocol.FileInfoTruncated
+			ef.UnmarshalXDR(dbi.Value())
+			if fs[fsi].Version > ef.Version || fs[fsi].Version != ef.Version {
+				if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
+					maxLocalVer = lv
+				}
+				if fs[fsi].IsInvalid() {
+					ldbRemoveFromGlobal(snap, batch, repo, node, newName)
+				} else {
+					ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+				}
+			}
+			// Iterate both sides.
+			fsi++
+			moreDb = dbi.Next()
+
+		case moreDb && (!moreFs || cmp == 1):
+			if deleteFn != nil {
+				if lv := deleteFn(snap, batch, repo, node, oldName, dbi); lv > maxLocalVer {
+					maxLocalVer = lv
+				}
+			}
+			moreDb = dbi.Next()
+		}
+	}
+
+	err = db.Write(batch, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	return maxLocalVer
+}
+
+func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
+	// TODO: Return the remaining maxLocalVer?
+	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
+		// Disk has files that we are missing. Remove it.
+		if debug {
+			l.Debugf("delete; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
+		}
+		ldbRemoveFromGlobal(db, batch, repo, node, name)
+		batch.Delete(dbi.Key())
+		return 0
+	})
+}
+
+func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
+	return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
+		var tf protocol.FileInfoTruncated
+		err := tf.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if !tf.IsDeleted() {
+			if debug {
+				l.Debugf("mark deleted; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
+			}
+			ts := clock(tf.LocalVersion)
+			f := protocol.FileInfo{
+				Name:         tf.Name,
+				Version:      lamport.Default.Tick(tf.Version),
+				LocalVersion: ts,
+				Flags:        tf.Flags | protocol.FlagDeleted,
+				Modified:     tf.Modified,
+			}
+			batch.Put(dbi.Key(), f.MarshalXDR())
+			ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
+			return ts
+		}
+		return 0
+	})
+}
+
+func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
+	runtime.GC()
+
+	batch := new(leveldb.Batch)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	var maxLocalVer uint64
+	for _, f := range fs {
+		name := []byte(f.Name)
+		fk := nodeKey(repo, node, name)
+		bs, err := snap.Get(fk, nil)
+		if err == leveldb.ErrNotFound {
+			if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
+				maxLocalVer = lv
+			}
+			if f.IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, name)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			}
+			continue
+		}
+
+		var ef protocol.FileInfoTruncated
+		err = ef.UnmarshalXDR(bs)
+		if err != nil {
+			panic(err)
+		}
+		// Flags might change without the version being bumped when we set the
+		// invalid flag on an existing file.
+		if ef.Version != f.Version || ef.Flags != f.Flags {
+			if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
+				maxLocalVer = lv
+			}
+			if f.IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, name)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			}
+		}
+	}
+
+	err = db.Write(batch, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	return maxLocalVer
+}
+
+func ldbInsert(batch dbWriter, repo, node, name []byte, file protocol.FileInfo) uint64 {
+	if debug {
+		l.Debugf("insert; repo=%q node=%v %v", repo, protocol.NodeIDFromBytes(node), file)
+	}
+
+	if file.LocalVersion == 0 {
+		file.LocalVersion = clock(0)
+	}
+
+	nk := nodeKey(repo, node, name)
+	batch.Put(nk, file.MarshalXDR())
+
+	return file.LocalVersion
+}
+
+// ldbUpdateGlobal adds this node+version to the version list for the given
+// file. If the node is already present in the list, the version is updated.
+// If the file does not have an entry in the global list, it is created.
+func ldbUpdateGlobal(db dbReader, batch dbWriter, repo, node, file []byte, version uint64) bool {
+	if debug {
+		l.Debugf("update global; repo=%q node=%v file=%q version=%d", repo, protocol.NodeIDFromBytes(node), file, version)
+	}
+	gk := globalKey(repo, file)
+	svl, err := db.Get(gk, nil)
+	if err != nil && err != leveldb.ErrNotFound {
+		panic(err)
+	}
+
+	var fl versionList
+	nv := fileVersion{
+		node:    node,
+		version: version,
+	}
+	if svl != nil {
+		err = fl.UnmarshalXDR(svl)
+		if err != nil {
+			panic(err)
+		}
+
+		for i := range fl.versions {
+			if bytes.Compare(fl.versions[i].node, node) == 0 {
+				if fl.versions[i].version == version {
+					// No need to do anything
+					return false
+				}
+				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+				break
+			}
+		}
+	}
+
+	for i := range fl.versions {
+		if fl.versions[i].version <= version {
+			t := append(fl.versions, fileVersion{})
+			copy(t[i+1:], t[i:])
+			t[i] = nv
+			fl.versions = t
+			goto done
+		}
+	}
+
+	fl.versions = append(fl.versions, nv)
+
+done:
+	batch.Put(gk, fl.MarshalXDR())
+
+	return true
+}
+
+// ldbRemoveFromGlobal removes the node from the global version list for the
+// given file. If the version list is empty after this, the file entry is
+// removed entirely.
+func ldbRemoveFromGlobal(db dbReader, batch dbWriter, repo, node, file []byte) {
+	if debug {
+		l.Debugf("remove from global; repo=%q node=%v file=%q", repo, protocol.NodeIDFromBytes(node), file)
+	}
+
+	gk := globalKey(repo, file)
+	svl, err := db.Get(gk, nil)
+	if err != nil {
+		// We might be called to "remove" a global version that doesn't exist
+		// if the first update for the file is already marked invalid.
+		return
+	}
+
+	var fl versionList
+	err = fl.UnmarshalXDR(svl)
+	if err != nil {
+		panic(err)
+	}
+
+	for i := range fl.versions {
+		if bytes.Compare(fl.versions[i].node, node) == 0 {
+			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+			break
+		}
+	}
+
+	if len(fl.versions) == 0 {
+		batch.Delete(gk)
+	} else {
+		batch.Put(gk, fl.MarshalXDR())
+	}
+}
+
+func ldbWithHave(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
+	start := nodeKey(repo, node, nil)                            // before all repo/node files
+	limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		f, err := unmarshalTrunc(dbi.Value(), truncate)
+		if err != nil {
+			panic(err)
+		}
+		if cont := fn(f); !cont {
+			return
+		}
+	}
+}
+
+func ldbWithAllRepoTruncated(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol.FileInfoTruncated) bool) {
+	runtime.GC()
+
+	start := nodeKey(repo, nil, nil)                                                // before all repo/node files
+	limit := nodeKey(repo, protocol.LocalNodeID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		node := nodeKeyNode(dbi.Key())
+		var f protocol.FileInfoTruncated
+		err := f.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if cont := fn(node, f); !cont {
+			return
+		}
+	}
+}
+
+func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
+	nk := nodeKey(repo, node, file)
+	bs, err := db.Get(nk, nil)
+	if err == leveldb.ErrNotFound {
+		return protocol.FileInfo{}
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var f protocol.FileInfo
+	err = f.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo {
+	k := globalKey(repo, file)
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	bs, err := snap.Get(k, nil)
+	if err == leveldb.ErrNotFound {
+		return protocol.FileInfo{}
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var vl versionList
+	err = vl.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	if len(vl.versions) == 0 {
+		l.Debugln(k)
+		panic("no versions?")
+	}
+
+	k = nodeKey(repo, vl.versions[0].node, file)
+	bs, err = snap.Get(k, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	var f protocol.FileInfo
+	err = f.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func ldbWithGlobal(db *leveldb.DB, repo []byte, truncate bool, fn fileIterator) {
+	runtime.GC()
+
+	start := globalKey(repo, nil)
+	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	for dbi.Next() {
+		var vl versionList
+		err := vl.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if len(vl.versions) == 0 {
+			l.Debugln(dbi.Key())
+			panic("no versions?")
+		}
+		fk := nodeKey(repo, vl.versions[0].node, globalKeyName(dbi.Key()))
+		bs, err := snap.Get(fk, nil)
+		if err != nil {
+			panic(err)
+		}
+
+		f, err := unmarshalTrunc(bs, truncate)
+		if err != nil {
+			panic(err)
+		}
+
+		if cont := fn(f); !cont {
+			return
+		}
+	}
+}
+
+func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
+	k := globalKey(repo, file)
+	bs, err := db.Get(k, nil)
+	if err == leveldb.ErrNotFound {
+		return nil
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	var vl versionList
+	err = vl.UnmarshalXDR(bs)
+	if err != nil {
+		panic(err)
+	}
+
+	var nodes []protocol.NodeID
+	for _, v := range vl.versions {
+		if v.version != vl.versions[0].version {
+			break
+		}
+		n := protocol.NodeIDFromBytes(v.node)
+		nodes = append(nodes, n)
+	}
+
+	return nodes
+}
+
+func ldbWithNeed(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
+	runtime.GC()
+
+	start := globalKey(repo, nil)
+	limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+outer:
+	for dbi.Next() {
+		var vl versionList
+		err := vl.UnmarshalXDR(dbi.Value())
+		if err != nil {
+			panic(err)
+		}
+		if len(vl.versions) == 0 {
+			l.Debugln(dbi.Key())
+			panic("no versions?")
+		}
+
+		have := false // If we have the file, any version
+		need := false // If we have a lower version of the file
+		var haveVersion uint64
+		for _, v := range vl.versions {
+			if bytes.Compare(v.node, node) == 0 {
+				have = true
+				haveVersion = v.version
+				need = v.version < vl.versions[0].version
+				break
+			}
+		}
+
+		if need || !have {
+			name := globalKeyName(dbi.Key())
+			needVersion := vl.versions[0].version
+		inner:
+			for i := range vl.versions {
+				if vl.versions[i].version != needVersion {
+					// We haven't found a valid copy of the file with the needed version.
+					continue outer
+				}
+				fk := nodeKey(repo, vl.versions[i].node, name)
+				bs, err := snap.Get(fk, nil)
+				if err != nil {
+					panic(err)
+				}
+
+				gf, err := unmarshalTrunc(bs, truncate)
+				if err != nil {
+					panic(err)
+				}
+
+				if gf.IsInvalid() {
+					// The file is marked invalid for whatever reason, don't use it.
+					continue inner
+				}
+
+				if gf.IsDeleted() && !have {
+					// We don't need deleted files that we don't have
+					continue outer
+				}
+
+				if debug {
+					l.Debugf("need repo=%q node=%v name=%q need=%v have=%v haveV=%d globalV=%d", repo, protocol.NodeIDFromBytes(node), name, need, have, haveVersion, vl.versions[0].version)
+				}
+
+				if cont := fn(gf); !cont {
+					return
+				}
+
+				// This file is handled, no need to look further in the version list
+				continue outer
+			}
+		}
+	}
+}
+
+func ldbListRepos(db *leveldb.DB) []string {
+	runtime.GC()
+
+	start := []byte{keyTypeGlobal}
+	limit := []byte{keyTypeGlobal + 1}
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	defer dbi.Release()
+
+	repoExists := make(map[string]bool)
+	for dbi.Next() {
+		repo := string(globalKeyRepo(dbi.Key()))
+		if !repoExists[repo] {
+			repoExists[repo] = true
+		}
+	}
+
+	repos := make([]string, 0, len(repoExists))
+	for k := range repoExists {
+		repos = append(repos, k)
+	}
+
+	sort.Strings(repos)
+	return repos
+}
+
+func ldbDropRepo(db *leveldb.DB, repo []byte) {
+	runtime.GC()
+
+	snap, err := db.GetSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	defer snap.Release()
+
+	// Remove all items related to the given repo from the node->file bucket
+	start := []byte{keyTypeNode}
+	limit := []byte{keyTypeNode + 1}
+	dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	for dbi.Next() {
+		itemRepo := nodeKeyRepo(dbi.Key())
+		if bytes.Compare(repo, itemRepo) == 0 {
+			db.Delete(dbi.Key(), nil)
+		}
+	}
+	dbi.Release()
+
+	// Remove all items related to the given repo from the global bucket
+	start = []byte{keyTypeGlobal}
+	limit = []byte{keyTypeGlobal + 1}
+	dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
+	for dbi.Next() {
+		itemRepo := globalKeyRepo(dbi.Key())
+		if bytes.Compare(repo, itemRepo) == 0 {
+			db.Delete(dbi.Key(), nil)
+		}
+	}
+	dbi.Release()
+}
+
+func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
+	if truncate {
+		var tf protocol.FileInfoTruncated
+		err := tf.UnmarshalXDR(bs)
+		return tf, err
+	} else {
+		var tf protocol.FileInfo
+		err := tf.UnmarshalXDR(bs)
+		return tf, err
+	}
+}

internal/files/leveldb_xdr.go

@@ -0,0 +1,145 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package files
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+fileVersion Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                       version (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of node                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    node (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct fileVersion {
+	unsigned hyper version;
+	opaque node<>;
+}
+
+*/
+
+func (o fileVersion) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o fileVersion) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o fileVersion) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o fileVersion) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(o.version)
+	xw.WriteBytes(o.node)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *fileVersion) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *fileVersion) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *fileVersion) decodeXDR(xr *xdr.Reader) error {
+	o.version = xr.ReadUint64()
+	o.node = xr.ReadBytes()
+	return xr.Error()
+}
+
+/*
+
+versionList Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                      Number of versions                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\              Zero or more fileVersion Structures              \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct versionList {
+	fileVersion versions<>;
+}
+
+*/
+
+func (o versionList) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o versionList) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o versionList) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o versionList) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(uint32(len(o.versions)))
+	for i := range o.versions {
+		_, err := o.versions[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *versionList) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *versionList) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *versionList) decodeXDR(xr *xdr.Reader) error {
+	_versionsSize := int(xr.ReadUint32())
+	o.versions = make([]fileVersion, _versionsSize)
+	for i := range o.versions {
+		(&o.versions[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}

internal/files/set.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package files provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+package files
+
+import (
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/lamport"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type fileRecord struct {
+	File   protocol.FileInfo
+	Usage  int
+	Global bool
+}
+
+type bitset uint64
+
+type Set struct {
+	localVersion map[protocol.NodeID]uint64
+	mutex        sync.Mutex
+	repo         string
+	db           *leveldb.DB
+}
+
+func NewSet(repo string, db *leveldb.DB) *Set {
+	var s = Set{
+		localVersion: make(map[protocol.NodeID]uint64),
+		repo:         repo,
+		db:           db,
+	}
+
+	var nodeID protocol.NodeID
+	ldbWithAllRepoTruncated(db, []byte(repo), func(node []byte, f protocol.FileInfoTruncated) bool {
+		copy(nodeID[:], node)
+		if f.LocalVersion > s.localVersion[nodeID] {
+			s.localVersion[nodeID] = f.LocalVersion
+		}
+		lamport.Default.Tick(f.Version)
+		return true
+	})
+	if debug {
+		l.Debugf("loaded localVersion for %q: %#v", repo, s.localVersion)
+	}
+	clock(s.localVersion[protocol.LocalNodeID])
+
+	return &s
+}
+
+func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) {
+	if debug {
+		l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
+	}
+	normalizeFilenames(fs)
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.localVersion[node] = ldbReplace(s.db, []byte(s.repo), node[:], fs)
+}
+
+func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) {
+	if debug {
+		l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
+	}
+	normalizeFilenames(fs)
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if lv := ldbReplaceWithDelete(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
+		s.localVersion[node] = lv
+	}
+}
+
+func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) {
+	if debug {
+		l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
+	}
+	normalizeFilenames(fs)
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if lv := ldbUpdate(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
+		s.localVersion[node] = lv
+	}
+}
+
+func (s *Set) WithNeed(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithNeed(%v)", s.repo, node)
+	}
+	ldbWithNeed(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithNeedTruncated(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithNeedTruncated(%v)", s.repo, node)
+	}
+	ldbWithNeed(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
+}
+
+func (s *Set) WithHave(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithHave(%v)", s.repo, node)
+	}
+	ldbWithHave(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithHaveTruncated(node protocol.NodeID, fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithHaveTruncated(%v)", s.repo, node)
+	}
+	ldbWithHave(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
+}
+
+func (s *Set) WithGlobal(fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithGlobal()", s.repo)
+	}
+	ldbWithGlobal(s.db, []byte(s.repo), false, nativeFileIterator(fn))
+}
+
+func (s *Set) WithGlobalTruncated(fn fileIterator) {
+	if debug {
+		l.Debugf("%s WithGlobalTruncated()", s.repo)
+	}
+	ldbWithGlobal(s.db, []byte(s.repo), true, nativeFileIterator(fn))
+}
+
+func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo {
+	f := ldbGet(s.db, []byte(s.repo), node[:], []byte(normalizedFilename(file)))
+	f.Name = nativeFilename(f.Name)
+	return f
+}
+
+func (s *Set) GetGlobal(file string) protocol.FileInfo {
+	f := ldbGetGlobal(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
+	f.Name = nativeFilename(f.Name)
+	return f
+}
+
+func (s *Set) Availability(file string) []protocol.NodeID {
+	return ldbAvailability(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
+}
+
+func (s *Set) LocalVersion(node protocol.NodeID) uint64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.localVersion[node]
+}
+
+// ListRepos returns the repository IDs seen in the database.
+func ListRepos(db *leveldb.DB) []string {
+	return ldbListRepos(db)
+}
+
+// DropRepo clears out all information related to the given repo from the
+// database.
+func DropRepo(db *leveldb.DB, repo string) {
+	ldbDropRepo(db, []byte(repo))
+}
+
+func normalizeFilenames(fs []protocol.FileInfo) {
+	for i := range fs {
+		fs[i].Name = normalizedFilename(fs[i].Name)
+	}
+}
+
+func nativeFileIterator(fn fileIterator) fileIterator {
+	return func(fi protocol.FileIntf) bool {
+		switch f := fi.(type) {
+		case protocol.FileInfo:
+			f.Name = nativeFilename(f.Name)
+			return fn(f)
+		case protocol.FileInfoTruncated:
+			f.Name = nativeFilename(f.Name)
+			return fn(f)
+		default:
+			panic("unknown interface type")
+		}
+	}
+}

internal/files/set_test.go

@@ -0,0 +1,913 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package files_test
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/files"
+	"github.com/syncthing/syncthing/internal/lamport"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+var remoteNode0, remoteNode1 protocol.NodeID
+
+func init() {
+	remoteNode0, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
+	remoteNode1, _ = protocol.NodeIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
+}
+
+func genBlocks(n int) []protocol.BlockInfo {
+	b := make([]protocol.BlockInfo, n)
+	for i := range b {
+		h := make([]byte, 32)
+		for j := range h {
+			h[j] = byte(i + j)
+		}
+		b[i].Size = uint32(i)
+		b[i].Hash = h
+	}
+	return b
+}
+
+func globalList(s *files.Set) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithGlobal(func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithHave(n, func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
+	var fs []protocol.FileInfo
+	s.WithNeed(n, func(fi protocol.FileIntf) bool {
+		f := fi.(protocol.FileInfo)
+		fs = append(fs, f)
+		return true
+	})
+	return fs
+}
+
+type fileList []protocol.FileInfo
+
+func (l fileList) Len() int {
+	return len(l)
+}
+
+func (l fileList) Less(a, b int) bool {
+	return l[a].Name < l[b].Name
+}
+
+func (l fileList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l fileList) String() string {
+	var b bytes.Buffer
+	b.WriteString("[]protocol.FileList{\n")
+	for _, f := range l {
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
+	}
+	b.WriteString("}")
+	return b.String()
+}
+
+func TestGlobalSet(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	local0 := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)},
+	}
+	local1 := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
+	}
+	localTot := fileList{
+		local0[0],
+		local0[1],
+		local0[2],
+		local0[3],
+		protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
+	}
+
+	remote0 := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)},
+	}
+	remote1 := fileList{
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)},
+	}
+	remoteTot := fileList{
+		remote0[0],
+		remote1[0],
+		remote0[2],
+		remote1[1],
+	}
+
+	expectedGlobal := fileList{
+		remote0[0],  // a
+		remote1[0],  // b
+		remote0[2],  // c
+		localTot[3], // d
+		remote1[1],  // e
+		localTot[4], // z
+	}
+
+	expectedLocalNeed := fileList{
+		remote1[0],
+		remote0[2],
+		remote1[1],
+	}
+
+	expectedRemoteNeed := fileList{
+		local0[3],
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local0)
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
+	m.Replace(remoteNode0, remote0)
+	m.Update(remoteNode0, remote1)
+
+	g := fileList(globalList(m))
+	sort.Sort(g)
+
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
+	}
+
+	h := fileList(haveList(m, protocol.LocalNodeID))
+	sort.Sort(h)
+
+	if fmt.Sprint(h) != fmt.Sprint(localTot) {
+		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
+	}
+
+	h = fileList(haveList(m, remoteNode0))
+	sort.Sort(h)
+
+	if fmt.Sprint(h) != fmt.Sprint(remoteTot) {
+		t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, remoteTot)
+	}
+
+	n := fileList(needList(m, protocol.LocalNodeID))
+	sort.Sort(n)
+
+	if fmt.Sprint(n) != fmt.Sprint(expectedLocalNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedLocalNeed)
+	}
+
+	n = fileList(needList(m, remoteNode0))
+	sort.Sort(n)
+
+	if fmt.Sprint(n) != fmt.Sprint(expectedRemoteNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
+	}
+
+	f := m.Get(protocol.LocalNodeID, "b")
+	if fmt.Sprint(f) != fmt.Sprint(localTot[1]) {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
+	}
+
+	f = m.Get(remoteNode0, "b")
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
+	}
+
+	f = m.GetGlobal("b")
+	if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
+	}
+
+	f = m.Get(protocol.LocalNodeID, "zz")
+	if f.Name != "" {
+		t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
+	}
+
+	f = m.GetGlobal("zz")
+	if f.Name != "" {
+		t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
+	}
+
+	av := []protocol.NodeID{protocol.LocalNodeID, remoteNode0}
+	a := m.Availability("a")
+	if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
+	}
+	a = m.Availability("b")
+	if len(a) != 1 || a[0] != remoteNode0 {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteNode0)
+	}
+	a = m.Availability("d")
+	if len(a) != 1 || a[0] != protocol.LocalNodeID {
+		t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalNodeID)
+	}
+}
+
+func TestNeedWithInvalid(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	localHave := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+	}
+	remote0Have := fileList{
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+	}
+	remote1Have := fileList{
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "e", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+
+	expectedNeed := fileList{
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+	}
+
+	s.ReplaceWithDelete(protocol.LocalNodeID, localHave)
+	s.Replace(remoteNode0, remote0Have)
+	s.Replace(remoteNode1, remote1Have)
+
+	need := fileList(needList(s, protocol.LocalNodeID))
+	sort.Sort(need)
+
+	if fmt.Sprint(need) != fmt.Sprint(expectedNeed) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed)
+	}
+}
+
+func TestUpdateToInvalid(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	localHave := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+	}
+
+	s.ReplaceWithDelete(protocol.LocalNodeID, localHave)
+
+	have := fileList(haveList(s, protocol.LocalNodeID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+
+	localHave[1] = protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagInvalid}
+	s.Update(protocol.LocalNodeID, localHave[1:2])
+
+	have = fileList(haveList(s, protocol.LocalNodeID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect after invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+}
+
+func TestInvalidAvailability(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	remote0Have := fileList{
+		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+	remote1Have := fileList{
+		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+
+	s.Replace(remoteNode0, remote0Have)
+	s.Replace(remoteNode1, remote1Have)
+
+	if av := s.Availability("both"); len(av) != 2 {
+		t.Error("Incorrect availability for 'both':", av)
+	}
+
+	if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteNode0 {
+		t.Error("Incorrect availability for 'r0only':", av)
+	}
+
+	if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteNode1 {
+		t.Error("Incorrect availability for 'r1only':", av)
+	}
+
+	if av := s.Availability("none"); len(av) != 0 {
+		t.Error("Incorrect availability for 'none':", av)
+	}
+}
+
+func TestLocalDeleted(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := files.NewSet("test", db)
+	lamport.Default = lamport.Clock{}
+
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
+		protocol.FileInfo{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
+		local1[0],
+		// [1] removed
+		local1[2],
+		local1[3],
+		local1[4],
+	})
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
+		local1[0],
+		local1[2],
+		// [3] removed
+		local1[4],
+	})
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
+		local1[0],
+		local1[2],
+		// [4] removed
+	})
+
+	expectedGlobal1 := []protocol.FileInfo{
+		local1[0],
+		protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
+		local1[2],
+		protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+	}
+
+	g := globalList(m)
+	sort.Sort(fileList(g))
+	sort.Sort(fileList(expectedGlobal1))
+
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal1) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
+		local1[0],
+		// [2] removed
+	})
+
+	expectedGlobal2 := []protocol.FileInfo{
+		local1[0],
+		protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
+	}
+
+	g = globalList(m)
+	sort.Sort(fileList(g))
+	sort.Sort(fileList(expectedGlobal2))
+
+	if fmt.Sprint(g) != fmt.Sprint(expectedGlobal2) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal2)
+	}
+}
+
+func Benchmark10kReplace(b *testing.B) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m := files.NewSet("test", db)
+		m.ReplaceWithDelete(protocol.LocalNodeID, local)
+	}
+}
+
+func Benchmark10kUpdateChg(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		for j := range local {
+			local[j].Version++
+		}
+		b.StartTimer()
+		m.Update(protocol.LocalNodeID, local)
+	}
+}
+
+func Benchmark10kUpdateSme(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Update(protocol.LocalNodeID, local)
+	}
+}
+
+func Benchmark10kNeed2k(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 8000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 8000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := needList(m, protocol.LocalNodeID)
+		if l := len(fs); l != 2000 {
+			b.Errorf("wrong length %d != 2k", l)
+		}
+	}
+}
+
+func Benchmark10kHaveFullList(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 2000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := haveList(m, protocol.LocalNodeID)
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
+func Benchmark10kGlobal(b *testing.B) {
+	var remote []protocol.FileInfo
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+	m.Replace(remoteNode0, remote)
+
+	var local []protocol.FileInfo
+	for i := 0; i < 2000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := globalList(m)
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
+func TestGlobalReset(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
+	}
+
+	remote := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+	g := globalList(m)
+	sort.Sort(fileList(g))
+
+	if fmt.Sprint(g) != fmt.Sprint(local) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
+	}
+
+	m.Replace(remoteNode0, remote)
+	m.Replace(remoteNode0, nil)
+
+	g = globalList(m)
+	sort.Sort(fileList(g))
+
+	if fmt.Sprint(g) != fmt.Sprint(local) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
+	}
+}
+
+func TestNeed(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
+	}
+
+	remote := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
+	}
+
+	shouldNeed := []protocol.FileInfo{
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local)
+	m.Replace(remoteNode0, remote)
+
+	need := needList(m, protocol.LocalNodeID)
+
+	sort.Sort(fileList(need))
+	sort.Sort(fileList(shouldNeed))
+
+	if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
+		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
+	}
+}
+
+func TestLocalVersion(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+		protocol.FileInfo{Name: "d", Version: 1000},
+	}
+
+	local2 := []protocol.FileInfo{
+		local1[0],
+		// [1] deleted
+		local1[2],
+		protocol.FileInfo{Name: "d", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1000},
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local1)
+	c0 := m.LocalVersion(protocol.LocalNodeID)
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local2)
+	c1 := m.LocalVersion(protocol.LocalNodeID)
+	if !(c1 > c0) {
+		t.Fatal("Local version number should have incremented")
+	}
+
+	m.ReplaceWithDelete(protocol.LocalNodeID, local2)
+	c2 := m.LocalVersion(protocol.LocalNodeID)
+	if c2 != c1 {
+		t.Fatal("Local version number should be unchanged")
+	}
+}
+
+func TestListDropRepo(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s0 := files.NewSet("test0", db)
+	local1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+		protocol.FileInfo{Name: "c", Version: 1000},
+	}
+	s0.Replace(protocol.LocalNodeID, local1)
+
+	s1 := files.NewSet("test1", db)
+	local2 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "d", Version: 1002},
+		protocol.FileInfo{Name: "e", Version: 1002},
+		protocol.FileInfo{Name: "f", Version: 1002},
+	}
+	s1.Replace(remoteNode0, local2)
+
+	// Check that we have both repos and their data is in the global list
+
+	expectedRepoList := []string{"test0", "test1"}
+	if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
+		t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s1", l)
+	}
+
+	// Drop one of them and check that it's gone.
+
+	files.DropRepo(db, "test1")
+
+	expectedRepoList = []string{"test0"}
+	if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
+		t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
+	}
+	if l := len(globalList(s0)); l != 3 {
+		t.Errorf("Incorrect global length %d != 3 for s0", l)
+	}
+	if l := len(globalList(s1)); l != 0 {
+		t.Errorf("Incorrect global length %d != 0 for s1", l)
+	}
+}
+
+func TestGlobalNeedWithInvalid(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test1", db)
+
+	rem0 := fileList{
+		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: 1002, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
+	}
+	s.Replace(remoteNode0, rem0)
+
+	rem1 := fileList{
+		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: 1002, Flags: protocol.FlagInvalid},
+	}
+	s.Replace(remoteNode1, rem1)
+
+	total := fileList{
+		// There's a valid copy of each file, so it should be merged
+		protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
+	}
+
+	need := fileList(needList(s, protocol.LocalNodeID))
+	if fmt.Sprint(need) != fmt.Sprint(total) {
+		t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total)
+	}
+
+	global := fileList(globalList(s))
+	if fmt.Sprint(global) != fmt.Sprint(total) {
+		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", global, total)
+	}
+}
+
+func TestLongPath(t *testing.T) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	var b bytes.Buffer
+	for i := 0; i < 100; i++ {
+		b.WriteString("012345678901234567890123456789012345678901234567890")
+	}
+	name := b.String() // 5000 characters
+
+	local := []protocol.FileInfo{
+		protocol.FileInfo{Name: string(name), Version: 1000},
+	}
+
+	s.ReplaceWithDelete(protocol.LocalNodeID, local)
+
+	gf := globalList(s)
+	if l := len(gf); l != 1 {
+		t.Fatalf("Incorrect len %d != 1 for global list", l)
+	}
+	if gf[0].Name != local[0].Name {
+		t.Error("Incorrect long filename;\n%q !=\n%q", gf[0].Name, local[0].Name)
+	}
+}
+
+/*
+var gf protocol.FileInfo
+
+func TestStressGlobalVersion(t *testing.T) {
+	dur := 15 * time.Second
+	if testing.Short() {
+		dur = 1 * time.Second
+	}
+
+	set1 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "a", Version: 1000},
+		protocol.FileInfo{Name: "b", Version: 1000},
+	}
+	set2 := []protocol.FileInfo{
+		protocol.FileInfo{Name: "b", Version: 1001},
+		protocol.FileInfo{Name: "c", Version: 1000},
+	}
+
+	db, err := leveldb.OpenFile("testdata/global.db", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := files.NewSet("test", db)
+
+	done := make(chan struct{})
+	go stressWriter(m, remoteNode0, set1, nil, done)
+	go stressWriter(m, protocol.LocalNodeID, set2, nil, done)
+
+	t0 := time.Now()
+	for time.Since(t0) < dur {
+		m.WithGlobal(func(f protocol.FileInfo) bool {
+			gf = f
+			return true
+		})
+	}
+
+	close(done)
+}
+
+func stressWriter(s *files.Set, id protocol.NodeID, set1, set2 []protocol.FileInfo, done chan struct{}) {
+	one := true
+	i := 0
+	for {
+		select {
+		case <-done:
+			return
+
+		default:
+			if one {
+				s.Replace(id, set1)
+			} else {
+				s.Replace(id, set2)
+			}
+			one = !one
+		}
+		i++
+	}
+}
+*/

internal/files/testdata/.gitignore

@@ -0,0 +1 @@
+index.db

internal/fnmatch/fnmatch.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+const (
+	FNM_NOESCAPE = (1 << iota)
+	FNM_PATHNAME
+	FNM_CASEFOLD
+)
+
+func Convert(pattern string, flags int) (*regexp.Regexp, error) {
+	any := "."
+
+	switch runtime.GOOS {
+	case "windows":
+		flags |= FNM_NOESCAPE | FNM_CASEFOLD
+		pattern = filepath.FromSlash(pattern)
+		if flags&FNM_PATHNAME != 0 {
+			any = "[^\\\\]"
+		}
+	case "darwin":
+		flags |= FNM_CASEFOLD
+		fallthrough
+	default:
+		if flags&FNM_PATHNAME != 0 {
+			any = "[^/]"
+		}
+	}
+
+	if flags&FNM_NOESCAPE != 0 {
+		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+	} else {
+		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+	}
+	pattern = strings.Replace(pattern, ".", "\\.", -1)
+	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
+	pattern = strings.Replace(pattern, "*", any+"*", -1)
+	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
+	pattern = strings.Replace(pattern, "?", any, -1)
+	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
+	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
+	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+	pattern = "^" + pattern + "$"
+	if flags&FNM_CASEFOLD != 0 {
+		pattern = "(?i)" + pattern
+	}
+	return regexp.Compile(pattern)
+}
+
+// Matches the pattern against the string, with the given flags,
+// and returns true if the match is successful.
+func Match(pattern, s string, flags int) (bool, error) {
+	exp, err := Convert(pattern, flags)
+	if err != nil {
+		return false, err
+	}
+	return exp.MatchString(s), nil
+}

internal/fnmatch/fnmatch_test.go

@@ -0,0 +1,86 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+type testcase struct {
+	pat   string
+	name  string
+	flags int
+	match bool
+}
+
+var testcases = []testcase{
+	{"", "", 0, true},
+	{"*", "", 0, true},
+	{"*", "foo", 0, true},
+	{"*", "bar", 0, true},
+	{"*", "*", 0, true},
+	{"**", "f", 0, true},
+	{"**", "foo.txt", 0, true},
+	{"*.*", "foo.txt", 0, true},
+	{"foo*.txt", "foobar.txt", 0, true},
+	{"foo.txt", "foo.txt", 0, true},
+
+	{"foo.txt", "bar/foo.txt", 0, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"f?o.txt", "foo.txt", 0, true},
+	{"f?o.txt", "fooo.txt", 0, false},
+	{"f[ab]o.txt", "foo.txt", 0, false},
+	{"f[ab]o.txt", "fao.txt", 0, true},
+	{"f[ab]o.txt", "fbo.txt", 0, true},
+	{"f[ab]o.txt", "fco.txt", 0, false},
+	{"f[ab]o.txt", "fabo.txt", 0, false},
+	{"f[ab]o.txt", "f[ab]o.txt", 0, false},
+	{"f\\[ab\\]o.txt", "f[ab]o.txt", FNM_NOESCAPE, false},
+
+	{"*foo.txt", "bar/foo.txt", 0, true},
+	{"*foo.txt", "bar/foo.txt", FNM_PATHNAME, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"*/foo.txt", "bar/foo.txt", FNM_PATHNAME, true},
+	{"*/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"*/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, false},
+	{"**/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"**/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, true},
+
+	{"foo.txt", "foo.TXT", FNM_CASEFOLD, true},
+}
+
+func TestMatch(t *testing.T) {
+	switch runtime.GOOS {
+	case "windows":
+		testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
+	case "darwin":
+		testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
+		fallthrough
+	default:
+		testcases = append(testcases, testcase{"f\\[ab\\]o.txt", "f[ab]o.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\*.txt", "foo*.txt", 0, true})
+		testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", FNM_NOESCAPE, false})
+		testcases = append(testcases, testcase{"f\\\\\\[ab\\\\\\]o.txt", "f\\[ab\\]o.txt", 0, true})
+	}
+
+	for _, tc := range testcases {
+		if m, err := Match(tc.pat, filepath.FromSlash(tc.name), tc.flags); m != tc.match {
+			if err != nil {
+				t.Error(err)
+			} else {
+				t.Errorf("Match(%q, %q, %d) != %v", tc.pat, tc.name, tc.flags, tc.match)
+			}
+		}
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	if _, err := Match("foo[bar", "...", 0); err == nil {
+		t.Error("Unexpected nil error")
+	}
+}

internal/ignore/ignore.go

@@ -0,0 +1,148 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package ignore
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/fnmatch"
+)
+
+type Pattern struct {
+	match   *regexp.Regexp
+	include bool
+}
+
+type Patterns []Pattern
+
+func Load(file string) (Patterns, error) {
+	seen := make(map[string]bool)
+	return loadIgnoreFile(file, seen)
+}
+
+func Parse(r io.Reader, file string) (Patterns, error) {
+	seen := map[string]bool{
+		file: true,
+	}
+	return parseIgnoreFile(r, file, seen)
+}
+
+func (l Patterns) Match(file string) bool {
+	for _, pattern := range l {
+		if pattern.match.MatchString(file) {
+			return pattern.include
+		}
+	}
+	return false
+}
+
+func loadIgnoreFile(file string, seen map[string]bool) (Patterns, error) {
+	if seen[file] {
+		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
+	}
+	seen[file] = true
+
+	fd, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	return parseIgnoreFile(fd, file, seen)
+}
+
+func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Patterns, error) {
+	var exps Patterns
+
+	addPattern := func(line string) error {
+		include := true
+		if strings.HasPrefix(line, "!") {
+			line = line[1:]
+			include = false
+		}
+
+		if strings.HasPrefix(line, "/") {
+			// Pattern is rooted in the current dir only
+			exp, err := fnmatch.Convert(line[1:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "**/") {
+			// Add the pattern as is, and without **/ so it matches in current dir
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert(line[3:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "#include ") {
+			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
+			includes, err := loadIgnoreFile(includeFile, seen)
+			if err != nil {
+				return err
+			} else {
+				exps = append(exps, includes...)
+			}
+		} else {
+			// Path name or pattern, add it so it matches files both in
+			// current directory and subdirs.
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert("**/"+line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		}
+		return nil
+	}
+
+	scanner := bufio.NewScanner(fd)
+	var err error
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case line == "":
+			continue
+		case strings.HasPrefix(line, "//"):
+			continue
+		case strings.HasPrefix(line, "#"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/**"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/"):
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "**")
+			}
+		default:
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "/**")
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return exps, nil
+}

internal/ignore/ignore_test.go

@@ -0,0 +1,153 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package ignore_test
+
+import (
+	"bytes"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/ignore"
+)
+
+func TestIgnore(t *testing.T) {
+	pats, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"afile", false},
+		{"bfile", true},
+		{"cfile", false},
+		{"dfile", false},
+		{"efile", true},
+		{"ffile", true},
+
+		{"dir1", false},
+		{filepath.Join("dir1", "cfile"), true},
+		{filepath.Join("dir1", "dfile"), false},
+		{filepath.Join("dir1", "efile"), true},
+		{filepath.Join("dir1", "ffile"), false},
+
+		{"dir2", false},
+		{filepath.Join("dir2", "cfile"), false},
+		{filepath.Join("dir2", "dfile"), true},
+		{filepath.Join("dir2", "efile"), true},
+		{filepath.Join("dir2", "ffile"), false},
+
+		{filepath.Join("dir3"), true},
+		{filepath.Join("dir3", "afile"), true},
+	}
+
+	for i, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
+		}
+	}
+}
+
+func TestExcludes(t *testing.T) {
+	stignore := `
+	!iex2
+	!ign1/ex
+	ign1
+	i*2
+	!ign2
+	`
+	pats, err := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"ign1", true},
+		{"ign2", true},
+		{"ibla2", true},
+		{"iex2", false},
+		{filepath.Join("ign1", "ign"), true},
+		{filepath.Join("ign1", "ex"), false},
+		{filepath.Join("ign1", "iex2"), false},
+		{filepath.Join("iex2", "ign"), false},
+		{filepath.Join("foo", "bar", "ign1"), true},
+		{filepath.Join("foo", "bar", "ign2"), true},
+		{filepath.Join("foo", "bar", "iex2"), false},
+	}
+
+	for _, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect match for %s: %v != %v", tc.f, r, tc.r)
+		}
+	}
+}
+
+func TestBadPatterns(t *testing.T) {
+	var badPatterns = []string{
+		"[",
+		"/[",
+		"**/[",
+		"#include nonexistent",
+		"#include .stignore",
+		"!#include makesnosense",
+	}
+
+	for _, pat := range badPatterns {
+		parsed, err := ignore.Parse(bytes.NewBufferString(pat), ".stignore")
+		if err == nil {
+			t.Errorf("No error for pattern %q: %v", pat, parsed)
+		}
+	}
+}
+
+func TestCaseSensitivity(t *testing.T) {
+	ign, _ := ignore.Parse(bytes.NewBufferString("test"), ".stignore")
+
+	match := []string{"test"}
+	dontMatch := []string{"foo"}
+
+	switch runtime.GOOS {
+	case "darwin", "windows":
+		match = append(match, "TEST", "Test", "tESt")
+	default:
+		dontMatch = append(dontMatch, "TEST", "Test", "tESt")
+	}
+
+	for _, tc := range match {
+		if !ign.Match(tc) {
+			t.Errorf("Incorrect match for %q: should be matched", tc)
+		}
+	}
+
+	for _, tc := range dontMatch {
+		if ign.Match(tc) {
+			t.Errorf("Incorrect match for %q: should not be matched", tc)
+		}
+	}
+}
+
+func TestCommentsAndBlankLines(t *testing.T) {
+	stignore := `
+	// foo
+	//bar
+
+	//!baz
+	//#dex
+
+	//                        ips
+
+
+	`
+	pats, _ := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if len(pats) > 0 {
+		t.Errorf("Expected no patterns")
+	}
+}

internal/ignore/testdata/.stignore

@@ -0,0 +1,6 @@
+#include excludes
+
+bfile
+dir1/cfile
+**/efile
+/ffile

internal/ignore/testdata/dir3/cfile

@@ -0,0 +1 @@
+baz

internal/ignore/testdata/dir3/dfile

@@ -0,0 +1 @@
+quux

internal/ignore/testdata/excludes

@@ -0,0 +1,2 @@
+dir2/dfile
+#include further-excludes

internal/ignore/testdata/further-excludes

@@ -0,0 +1 @@
+dir3

internal/lamport/clock.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package lamport implements a simple Lamport Clock for versioning
+package lamport
+
+import "sync"
+
+var Default = Clock{}
+
+type Clock struct {
+	val uint64
+	mut sync.Mutex
+}
+
+func (c *Clock) Tick(v uint64) uint64 {
+	c.mut.Lock()
+	if v > c.val {
+		c.val = v + 1
+		c.mut.Unlock()
+		return v + 1
+	} else {
+		c.val++
+		v = c.val
+		c.mut.Unlock()
+		return v
+	}
+}

internal/lamport/clock_test.go

@@ -0,0 +1,22 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package lamport
+
+import "testing"
+
+var inputs = []uint64{0, 42, 2, 3, 4, 8, 9, 33, 44, 112, 100}
+
+func TestClock(t *testing.T) {
+	c := Clock{}
+
+	var prev uint64
+	for _, input := range inputs {
+		cur := c.Tick(input)
+		if cur <= prev || cur <= input {
+			t.Error("Clock moving backwards")
+		}
+		prev = cur
+	}
+}

internal/logger/logger.go

@@ -0,0 +1,142 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package logger implements a standardized logger with callback functionality
+package logger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelInfo
+	LevelOK
+	LevelWarn
+	LevelFatal
+	NumLevels
+)
+
+type MessageHandler func(l LogLevel, msg string)
+
+type Logger struct {
+	logger   *log.Logger
+	handlers [NumLevels][]MessageHandler
+	mut      sync.Mutex
+}
+
+var DefaultLogger = New()
+
+func New() *Logger {
+	return &Logger{
+		logger: log.New(os.Stdout, "", log.Ltime),
+	}
+}
+
+func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	l.handlers[level] = append(l.handlers[level], h)
+}
+
+func (l *Logger) SetFlags(flag int) {
+	l.logger.SetFlags(flag)
+}
+
+func (l *Logger) SetPrefix(prefix string) {
+	l.logger.SetPrefix(prefix)
+}
+
+func (l *Logger) callHandlers(level LogLevel, s string) {
+	for _, h := range l.handlers[level] {
+		h(level, strings.TrimSpace(s))
+	}
+}
+
+func (l *Logger) Debugln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+func (l *Logger) Debugf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+func (l *Logger) Infoln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Infof(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Okln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Okf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Warnln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Warnf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Fatalln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}
+
+func (l *Logger) Fatalf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}

internal/logger/logger_test.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package logger
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestAPI(t *testing.T) {
+	l := New()
+	l.SetFlags(0)
+	l.SetPrefix("testing")
+
+	debug := 0
+	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
+	info := 0
+	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
+	warn := 0
+	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
+	ok := 0
+	l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
+
+	l.Debugf("test %d", 0)
+	l.Debugln("test", 0)
+	l.Infof("test %d", 1)
+	l.Infoln("test", 1)
+	l.Warnf("test %d", 2)
+	l.Warnln("test", 2)
+	l.Okf("test %d", 3)
+	l.Okln("test", 3)
+
+	if debug != 2 {
+		t.Errorf("Debug handler called %d != 2 times", debug)
+	}
+	if info != 2 {
+		t.Errorf("Info handler called %d != 2 times", info)
+	}
+	if warn != 2 {
+		t.Errorf("Warn handler called %d != 2 times", warn)
+	}
+	if ok != 2 {
+		t.Errorf("Ok handler called %d != 2 times", ok)
+	}
+}
+
+func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
+	return func(l LogLevel, msg string) {
+		*counter++
+		if l != expectl {
+			t.Errorf("Incorrect message level %d != %d", l, expectl)
+		}
+		if !strings.HasSuffix(msg, expectmsg) {
+			t.Errorf("%q does not end with %q", msg, expectmsg)
+		}
+	}
+}

internal/luhn/luhn.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package luhn generates and validates Luhn mod N check digits.
+package luhn
+
+import (
+	"fmt"
+	"strings"
+)
+
+// An alphabet is a string of N characters, representing the digits of a given
+// base N.
+type Alphabet string
+
+var (
+	Base32 Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+)
+
+// Generate returns a check digit for the string s, which should be composed
+// of characters from the Alphabet a.
+func (a Alphabet) Generate(s string) (rune, error) {
+	if err := a.check(); err != nil {
+		return 0, err
+	}
+
+	factor := 1
+	sum := 0
+	n := len(a)
+
+	for i := range s {
+		codepoint := strings.IndexByte(string(a), s[i])
+		if codepoint == -1 {
+			return 0, fmt.Errorf("Digit %q not valid in alphabet %q", s[i], a)
+		}
+		addend := factor * codepoint
+		if factor == 2 {
+			factor = 1
+		} else {
+			factor = 2
+		}
+		addend = (addend / n) + (addend % n)
+		sum += addend
+	}
+	remainder := sum % n
+	checkCodepoint := (n - remainder) % n
+	return rune(a[checkCodepoint]), nil
+}
+
+// Validate returns true if the last character of the string s is correct, for
+// a string s composed of characters in the alphabet a.
+func (a Alphabet) Validate(s string) bool {
+	t := s[:len(s)-1]
+	c, err := a.Generate(t)
+	if err != nil {
+		return false
+	}
+	return rune(s[len(s)-1]) == c
+}
+
+// check returns an error if the given alphabet does not consist of unique characters
+func (a Alphabet) check() error {
+	cm := make(map[byte]bool, len(a))
+	for i := range a {
+		if cm[a[i]] {
+			return fmt.Errorf("Digit %q non-unique in alphabet %q", a[i], a)
+		}
+		cm[a[i]] = true
+	}
+	return nil
+}

internal/luhn/luhn_test.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package luhn_test
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/luhn"
+)
+
+func TestGenerate(t *testing.T) {
+	// Base 6 Luhn
+	a := luhn.Alphabet("abcdef")
+	c, err := a.Generate("abcdef")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c != 'e' {
+		t.Errorf("Incorrect check digit %c != e", c)
+	}
+
+	// Base 10 Luhn
+	a = luhn.Alphabet("0123456789")
+	c, err = a.Generate("7992739871")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c != '3' {
+		t.Errorf("Incorrect check digit %c != 3", c)
+	}
+}
+
+func TestInvalidString(t *testing.T) {
+	a := luhn.Alphabet("ABC")
+	_, err := a.Generate("7992739871")
+	t.Log(err)
+	if err == nil {
+		t.Error("Unexpected nil error")
+	}
+}
+
+func TestBadAlphabet(t *testing.T) {
+	a := luhn.Alphabet("01234566789")
+	_, err := a.Generate("7992739871")
+	t.Log(err)
+	if err == nil {
+		t.Error("Unexpected nil error")
+	}
+}
+
+func TestValidate(t *testing.T) {
+	a := luhn.Alphabet("abcdef")
+	if !a.Validate("abcdefe") {
+		t.Errorf("Incorrect validation response for abcdefe")
+	}
+	if a.Validate("abcdefd") {
+		t.Errorf("Incorrect validation response for abcdefd")
+	}
+}

internal/model/blockqueue.go

@@ -0,0 +1,73 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import "github.com/syncthing/syncthing/internal/protocol"
+
+type bqAdd struct {
+	file protocol.FileInfo
+	have []protocol.BlockInfo
+	need []protocol.BlockInfo
+}
+
+type bqBlock struct {
+	file  protocol.FileInfo
+	block protocol.BlockInfo   // get this block from the network
+	copy  []protocol.BlockInfo // copy these blocks from the old version of the file
+	first bool
+	last  bool
+}
+
+type blockQueue struct {
+	queued []bqBlock
+}
+
+func (q *blockQueue) put(a bqAdd) {
+	// If we already have it queued, return
+	for _, b := range q.queued {
+		if b.file.Name == a.file.Name {
+			return
+		}
+	}
+
+	l := len(a.need)
+
+	if len(a.have) > 0 {
+		// First queue a copy operation
+		q.queued = append(q.queued, bqBlock{
+			file:  a.file,
+			copy:  a.have,
+			first: true,
+			last:  l == 0,
+		})
+	}
+
+	// Queue the needed blocks individually
+	for i, b := range a.need {
+		q.queued = append(q.queued, bqBlock{
+			file:  a.file,
+			block: b,
+			first: len(a.have) == 0 && i == 0,
+			last:  i == l-1,
+		})
+	}
+
+	if len(a.need)+len(a.have) == 0 {
+		// If we didn't have anything to fetch, queue an empty block with the "last" flag set to close the file.
+		q.queued = append(q.queued, bqBlock{
+			file: a.file,
+			last: true,
+		})
+	}
+}
+
+func (q *blockQueue) get() (bqBlock, bool) {
+	if len(q.queued) == 0 {
+		return bqBlock{}, false
+	}
+	b := q.queued[0]
+	q.queued = q.queued[1:]
+	return b, true
+}

internal/model/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/model/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package model implements repository abstraction and file pulling mechanisms
+package model

internal/model/model_test.go

@@ -0,0 +1,470 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+var node1, node2 protocol.NodeID
+
+func init() {
+	node1, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
+	node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
+}
+
+var testDataExpected = map[string]protocol.FileInfo{
+	"foo": protocol.FileInfo{
+		Name:     "foo",
+		Flags:    0,
+		Modified: 0,
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
+	},
+	"empty": protocol.FileInfo{
+		Name:     "empty",
+		Flags:    0,
+		Modified: 0,
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
+	},
+	"bar": protocol.FileInfo{
+		Name:     "bar",
+		Flags:    0,
+		Modified: 0,
+		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
+	},
+}
+
+func init() {
+	// Fix expected test data to match reality
+	for n, f := range testDataExpected {
+		fi, _ := os.Stat("testdata/" + n)
+		f.Flags = uint32(fi.Mode())
+		f.Modified = fi.ModTime().Unix()
+		testDataExpected[n] = f
+	}
+}
+
+func TestRequest(t *testing.T) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", &config.Configuration{}, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+
+	bs, err := m.Request(node1, "default", "foo", 0, 6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(bs, []byte("foobar")) != 0 {
+		t.Errorf("Incorrect data from request: %q", string(bs))
+	}
+
+	bs, err = m.Request(node1, "default", "../walk.go", 0, 6)
+	if err == nil {
+		t.Error("Unexpected nil error on insecure file read")
+	}
+	if bs != nil {
+		t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs))
+	}
+}
+
+func genFiles(n int) []protocol.FileInfo {
+	files := make([]protocol.FileInfo, n)
+	t := time.Now().Unix()
+	for i := 0; i < n; i++ {
+		files[i] = protocol.FileInfo{
+			Name:     fmt.Sprintf("file%d", i),
+			Modified: t,
+			Blocks:   []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
+		}
+	}
+
+	return files
+}
+
+func BenchmarkIndex10000(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+	files := genFiles(10000)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Index(node1, "default", files)
+	}
+}
+
+func BenchmarkIndex00100(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+	files := genFiles(100)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Index(node1, "default", files)
+	}
+}
+
+func BenchmarkIndexUpdate10000f10000(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index(node1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate(node1, "default", files)
+	}
+}
+
+func BenchmarkIndexUpdate10000f00100(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index(node1, "default", files)
+
+	ufiles := genFiles(100)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate(node1, "default", ufiles)
+	}
+}
+
+func BenchmarkIndexUpdate10000f00001(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index(node1, "default", files)
+
+	ufiles := genFiles(1)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate(node1, "default", ufiles)
+	}
+}
+
+type FakeConnection struct {
+	id          protocol.NodeID
+	requestData []byte
+}
+
+func (FakeConnection) Close() error {
+	return nil
+}
+
+func (f FakeConnection) ID() protocol.NodeID {
+	return f.id
+}
+
+func (f FakeConnection) Name() string {
+	return ""
+}
+
+func (f FakeConnection) Option(string) string {
+	return ""
+}
+
+func (FakeConnection) Index(string, []protocol.FileInfo) error {
+	return nil
+}
+
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
+	return nil
+}
+
+func (f FakeConnection) Request(repo, name string, offset int64, size int) ([]byte, error) {
+	return f.requestData, nil
+}
+
+func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+
+func (FakeConnection) Ping() bool {
+	return true
+}
+
+func (FakeConnection) Statistics() protocol.Statistics {
+	return protocol.Statistics{}
+}
+
+func BenchmarkRequest(b *testing.B) {
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+	m.ScanRepo("default")
+
+	const n = 1000
+	files := make([]protocol.FileInfo, n)
+	t := time.Now().Unix()
+	for i := 0; i < n; i++ {
+		files[i] = protocol.FileInfo{
+			Name:     fmt.Sprintf("file%d", i),
+			Modified: t,
+			Blocks:   []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
+		}
+	}
+
+	fc := FakeConnection{
+		id:          node1,
+		requestData: []byte("some data to return"),
+	}
+	m.AddConnection(fc, fc)
+	m.Index(node1, "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		data, err := m.requestGlobal(node1, "default", files[i%n].Name, 0, 32, nil)
+		if err != nil {
+			b.Error(err)
+		}
+		if data == nil {
+			b.Error("nil data")
+		}
+	}
+}
+
+func TestActivityMap(t *testing.T) {
+	isValid := func(protocol.NodeID) bool {
+		return true
+	}
+	m := make(activityMap)
+	if node := m.leastBusyNode([]protocol.NodeID{node1}, isValid); node != node1 {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode([]protocol.NodeID{node2}, isValid); node != node2 {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode([]protocol.NodeID{node1, node2}, isValid); node != node1 {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode([]protocol.NodeID{node1, node2}, isValid); node != node2 {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+}
+
+func TestNodeRename(t *testing.T) {
+	ccm := protocol.ClusterConfigMessage{
+		ClientName:    "syncthing",
+		ClientVersion: "v0.9.4",
+	}
+
+	cfg := config.New("/tmp/test", node1)
+	cfg.Nodes = []config.NodeConfiguration{
+		{
+			NodeID: node1,
+		},
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", &cfg, "node", "syncthing", "dev", db)
+	if cfg.Nodes[0].Name != "" {
+		t.Errorf("Node already has a name")
+	}
+
+	m.ClusterConfig(node1, ccm)
+	if cfg.Nodes[0].Name != "" {
+		t.Errorf("Node already has a name")
+	}
+
+	ccm.Options = []protocol.Option{
+		{
+			Key:   "name",
+			Value: "tester",
+		},
+	}
+	m.ClusterConfig(node1, ccm)
+	if cfg.Nodes[0].Name != "tester" {
+		t.Errorf("Node did not get a name")
+	}
+
+	ccm.Options[0].Value = "tester2"
+	m.ClusterConfig(node1, ccm)
+	if cfg.Nodes[0].Name != "tester" {
+		t.Errorf("Node name got overwritten")
+	}
+}
+
+func TestClusterConfig(t *testing.T) {
+	cfg := config.New("/tmp/test", node1)
+	cfg.Nodes = []config.NodeConfiguration{
+		{
+			NodeID:     node1,
+			Introducer: true,
+		},
+		{
+			NodeID: node2,
+		},
+	}
+	cfg.Repositories = []config.RepositoryConfiguration{
+		{
+			ID: "repo1",
+			Nodes: []config.RepositoryNodeConfiguration{
+				{NodeID: node1},
+				{NodeID: node2},
+			},
+		},
+		{
+			ID: "repo2",
+			Nodes: []config.RepositoryNodeConfiguration{
+				{NodeID: node1},
+				{NodeID: node2},
+			},
+		},
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	m := NewModel("/tmp", &cfg, "node", "syncthing", "dev", db)
+	m.AddRepo(cfg.Repositories[0])
+	m.AddRepo(cfg.Repositories[1])
+
+	cm := m.clusterConfig(node2)
+
+	if l := len(cm.Repositories); l != 2 {
+		t.Fatalf("Incorrect number of repos %d != 2", l)
+	}
+
+	r := cm.Repositories[0]
+	if r.ID != "repo1" {
+		t.Errorf("Incorrect repo %q != repo1", r.ID)
+	}
+	if l := len(r.Nodes); l != 2 {
+		t.Errorf("Incorrect number of nodes %d != 2", l)
+	}
+	if id := r.Nodes[0].ID; bytes.Compare(id, node1[:]) != 0 {
+		t.Errorf("Incorrect node ID %x != %x", id, node1)
+	}
+	if r.Nodes[0].Flags&protocol.FlagIntroducer == 0 {
+		t.Error("Node1 should be flagged as Introducer")
+	}
+	if id := r.Nodes[1].ID; bytes.Compare(id, node2[:]) != 0 {
+		t.Errorf("Incorrect node ID %x != %x", id, node2)
+	}
+	if r.Nodes[1].Flags&protocol.FlagIntroducer != 0 {
+		t.Error("Node2 should not be flagged as Introducer")
+	}
+
+	r = cm.Repositories[1]
+	if r.ID != "repo2" {
+		t.Errorf("Incorrect repo %q != repo2", r.ID)
+	}
+	if l := len(r.Nodes); l != 2 {
+		t.Errorf("Incorrect number of nodes %d != 2", l)
+	}
+	if id := r.Nodes[0].ID; bytes.Compare(id, node1[:]) != 0 {
+		t.Errorf("Incorrect node ID %x != %x", id, node1)
+	}
+	if r.Nodes[0].Flags&protocol.FlagIntroducer == 0 {
+		t.Error("Node1 should be flagged as Introducer")
+	}
+	if id := r.Nodes[1].ID; bytes.Compare(id, node2[:]) != 0 {
+		t.Errorf("Incorrect node ID %x != %x", id, node2)
+	}
+	if r.Nodes[1].Flags&protocol.FlagIntroducer != 0 {
+		t.Error("Node2 should not be flagged as Introducer")
+	}
+}
+
+func TestIgnores(t *testing.T) {
+	arrEqual := func(a, b []string) bool {
+		if len(a) != len(b) {
+			return false
+		}
+
+		for i := range a {
+			if a[i] != b[i] {
+				return false
+			}
+		}
+		return true
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel("/tmp", nil, "node", "syncthing", "dev", db)
+	m.AddRepo(config.RepositoryConfiguration{ID: "default", Directory: "testdata"})
+
+	expected := []string{
+		".*",
+		"quux",
+	}
+
+	ignores, err := m.GetIgnores("default")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !arrEqual(ignores, expected) {
+		t.Errorf("Incorrect ignores: %v != %v", ignores, expected)
+	}
+
+	ignores = append(ignores, "pox")
+
+	err = m.SetIgnores("default", ignores)
+	if err != nil {
+		t.Error(err)
+	}
+
+	ignores2, err := m.GetIgnores("default")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if arrEqual(expected, ignores2) {
+		t.Errorf("Incorrect ignores: %v == %v", ignores2, expected)
+	}
+
+	if !arrEqual(ignores, ignores2) {
+		t.Errorf("Incorrect ignores: %v != %v", ignores2, ignores)
+	}
+
+	err = m.SetIgnores("default", expected)
+	if err != nil {
+		t.Error(err)
+	}
+
+	ignores, err = m.GetIgnores("default")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !arrEqual(ignores, expected) {
+		t.Errorf("Incorrect ignores: %v != %v", ignores, expected)
+	}
+
+	ignores, err = m.GetIgnores("doesnotexist")
+	if err == nil {
+		t.Error("No error")
+	}
+
+	err = m.SetIgnores("doesnotexist", expected)
+	if err == nil {
+		t.Error("No error")
+	}
+
+	m.AddRepo(config.RepositoryConfiguration{ID: "fresh", Directory: "XXX"})
+	ignores, err = m.GetIgnores("fresh")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(ignores) > 0 {
+		t.Errorf("Expected no ignores, got: %v", ignores)
+	}
+}

internal/model/puller.go

@@ -0,0 +1,827 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+/*
+__        __               _             _
+\ \      / /_ _ _ __ _ __ (_)_ __   __ _| |
+ \ \ /\ / / _` | '__| '_ \| | '_ \ / _` | |
+  \ V  V / (_| | |  | | | | | | | | (_| |_|
+   \_/\_/ \__,_|_|  |_| |_|_|_| |_|\__, (_)
+                                   |___/
+
+The code in this file is a piece of crap. Don't base anything on it.
+Refactorin ongoing in new-puller branch.
+
+__        __               _             _
+\ \      / /_ _ _ __ _ __ (_)_ __   __ _| |
+ \ \ /\ / / _` | '__| '_ \| | '_ \ / _` | |
+  \ V  V / (_| | |  | | | | | | | | (_| |_|
+   \_/\_/ \__,_|_|  |_| |_|_|_| |_|\__, (_)
+                                   |___/
+*/
+
+package model
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syncthing/syncthing/internal/scanner"
+	"github.com/syncthing/syncthing/internal/versioner"
+)
+
+type requestResult struct {
+	node     protocol.NodeID
+	file     protocol.FileInfo
+	filepath string // full filepath name
+	offset   int64
+	data     []byte
+	err      error
+}
+
+type openFile struct {
+	filepath     string // full filepath name
+	temp         string // temporary filename
+	availability []protocol.NodeID
+	file         *os.File
+	err          error // error when opening or writing to file, all following operations are cancelled
+	outstanding  int   // number of requests we still have outstanding
+	done         bool  // we have sent all requests for this file
+}
+
+type activityMap map[protocol.NodeID]int
+
+// Queue about this many blocks each puller iteration. More blocks means
+// longer iterations and better efficiency; fewer blocks reduce memory
+// consumption. 1000 blocks ~= 1000 * 128 KiB ~= 125 MiB of data.
+const pullIterationBlocks = 1000
+
+func (m activityMap) leastBusyNode(availability []protocol.NodeID, isValid func(protocol.NodeID) bool) protocol.NodeID {
+	var low int = 2<<30 - 1
+	var selected protocol.NodeID
+	for _, node := range availability {
+		usage := m[node]
+		if usage < low && isValid(node) {
+			low = usage
+			selected = node
+		}
+	}
+	m[selected]++
+	return selected
+}
+
+func (m activityMap) decrease(node protocol.NodeID) {
+	m[node]--
+}
+
+var errNoNode = errors.New("no available source node")
+
+type puller struct {
+	cfg               *config.Configuration
+	repoCfg           config.RepositoryConfiguration
+	bq                blockQueue
+	slots             int
+	model             *Model
+	oustandingPerNode activityMap
+	openFiles         map[string]openFile
+	requestSlots      chan bool
+	blocks            chan bqBlock
+	requestResults    chan requestResult
+	versioner         versioner.Versioner
+	errors            int
+}
+
+func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int, cfg *config.Configuration) *puller {
+	p := &puller{
+		cfg:               cfg,
+		repoCfg:           repoCfg,
+		slots:             slots,
+		model:             model,
+		oustandingPerNode: make(activityMap),
+		openFiles:         make(map[string]openFile),
+		requestSlots:      make(chan bool, slots),
+		blocks:            make(chan bqBlock),
+		requestResults:    make(chan requestResult),
+	}
+
+	if len(repoCfg.Versioning.Type) > 0 {
+		factory, ok := versioner.Factories[repoCfg.Versioning.Type]
+		if !ok {
+			l.Fatalf("Requested versioning type %q that does not exist", repoCfg.Versioning.Type)
+		}
+		p.versioner = factory(repoCfg.ID, repoCfg.Directory, repoCfg.Versioning.Params)
+	}
+
+	if slots > 0 {
+		// Read/write
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q slots %d", repoCfg.ID, repoCfg.Directory, slots)
+		}
+		go p.run()
+	} else {
+		// Read only
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q (read only)", repoCfg.ID, repoCfg.Directory)
+		}
+		go p.runRO()
+	}
+	return p
+}
+
+func (p *puller) run() {
+	changed := true
+	scanintv := time.Duration(p.repoCfg.RescanIntervalS) * time.Second
+	lastscan := time.Now()
+	var prevVer uint64
+	var queued int
+
+	// Load up the request slots
+	for i := 0; i < cap(p.requestSlots); i++ {
+		p.requestSlots <- true
+	}
+
+	for {
+		if sc, sl := cap(p.requestSlots), len(p.requestSlots); sl != sc {
+			panic(fmt.Sprintf("Incorrect number of slots; %d != %d", sl, sc))
+		}
+
+		// Run the pulling loop as long as there are blocks to fetch
+		prevVer, queued = p.queueNeededBlocks(prevVer)
+		if queued > 0 {
+			p.errors = 0
+
+		pull:
+			for {
+				select {
+				case res := <-p.requestResults:
+					p.model.setState(p.repoCfg.ID, RepoSyncing)
+					changed = true
+					p.requestSlots <- true
+					p.handleRequestResult(res)
+
+				case <-p.requestSlots:
+					b, ok := p.bq.get()
+
+					if !ok {
+						if debug {
+							l.Debugf("%q: pulling loop needs more blocks", p.repoCfg.ID)
+						}
+
+						if p.errors > 0 && p.errors >= queued {
+							p.requestSlots <- true
+							break pull
+						}
+
+						prevVer, _ = p.queueNeededBlocks(prevVer)
+						b, ok = p.bq.get()
+					}
+
+					if !ok && len(p.openFiles) == 0 {
+						// Nothing queued, nothing outstanding
+						if debug {
+							l.Debugf("%q: pulling loop done", p.repoCfg.ID)
+						}
+						p.requestSlots <- true
+						break pull
+					}
+
+					if !ok {
+						// Nothing queued, but there are still open files.
+						// Give the situation a moment to change.
+						if debug {
+							l.Debugf("%q: pulling loop paused", p.repoCfg.ID)
+						}
+						p.requestSlots <- true
+						time.Sleep(100 * time.Millisecond)
+						continue pull
+					}
+
+					if debug {
+						l.Debugf("queueing %q / %q offset %d copy %d", p.repoCfg.ID, b.file.Name, b.block.Offset, len(b.copy))
+					}
+					p.model.setState(p.repoCfg.ID, RepoSyncing)
+					changed = true
+					if p.handleBlock(b) {
+						// Block was fully handled, free up the slot
+						p.requestSlots <- true
+					}
+				}
+			}
+
+			if p.errors > 0 && p.errors >= queued {
+				l.Warnf("All remaining files failed to sync. Stopping repo %q.", p.repoCfg.ID)
+				invalidateRepo(p.cfg, p.repoCfg.ID, errors.New("too many errors, check logs"))
+				return
+			}
+		}
+
+		if changed {
+			p.model.setState(p.repoCfg.ID, RepoCleaning)
+			p.clean()
+			changed = false
+		}
+
+		p.model.setState(p.repoCfg.ID, RepoIdle)
+
+		// Do a rescan if it's time for it
+		if time.Since(lastscan) > scanintv {
+			if debug {
+				l.Debugf("%q: time for rescan", p.repoCfg.ID)
+			}
+
+			err := p.model.ScanRepo(p.repoCfg.ID)
+			if err != nil {
+				invalidateRepo(p.cfg, p.repoCfg.ID, err)
+				return
+			}
+			lastscan = time.Now()
+		}
+
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func (p *puller) runRO() {
+	walkTicker := time.Tick(time.Duration(p.repoCfg.RescanIntervalS) * time.Second)
+
+	for _ = range walkTicker {
+		if debug {
+			l.Debugf("%q: time for rescan", p.repoCfg.ID)
+		}
+		err := p.model.ScanRepo(p.repoCfg.ID)
+		if err != nil {
+			invalidateRepo(p.cfg, p.repoCfg.ID, err)
+			return
+		}
+	}
+}
+
+// clean deletes orphaned temporary files and directories that should no
+// longer exist.
+func (p *puller) clean() {
+	var deleteDirs []string
+	var changed = 0
+
+	var walkFn = func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.Mode().IsRegular() && defTempNamer.IsTemporary(path) {
+			os.Remove(path)
+		}
+
+		if !info.IsDir() {
+			return nil
+		}
+
+		rn, err := filepath.Rel(p.repoCfg.Directory, path)
+		if err != nil {
+			return nil
+		}
+
+		if rn == "." {
+			return nil
+		}
+
+		if filepath.Base(rn) == ".stversions" {
+			return filepath.SkipDir
+		}
+
+		cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
+		if cur.Name != rn {
+			// No matching dir in current list; weird
+			if debug {
+				l.Debugf("missing dir: %s; %v", rn, cur)
+			}
+			return nil
+		}
+
+		if protocol.IsDeleted(cur.Flags) {
+			if debug {
+				l.Debugf("queue delete dir: %v", cur)
+			}
+
+			// We queue the directories to delete since we walk the
+			// tree in depth first order and need to remove the
+			// directories in the opposite order.
+
+			deleteDirs = append(deleteDirs, path)
+			return nil
+		}
+
+		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
+			err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
+			if err != nil {
+				l.Warnf("Restoring folder flags: %q: %v", path, err)
+			} else {
+				changed++
+				if debug {
+					l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
+				}
+			}
+		}
+
+		return nil
+	}
+
+	for {
+		deleteDirs = nil
+		changed = 0
+		filepath.Walk(p.repoCfg.Directory, walkFn)
+
+		var deleted = 0
+		// Delete any queued directories
+		for i := len(deleteDirs) - 1; i >= 0; i-- {
+			dir := deleteDirs[i]
+			if debug {
+				l.Debugln("delete dir:", dir)
+			}
+			err := os.Remove(dir)
+			if err == nil {
+				deleted++
+			} else {
+				l.Warnln("Delete dir:", err)
+			}
+		}
+
+		if debug {
+			l.Debugf("changed %d, deleted %d dirs", changed, deleted)
+		}
+
+		if changed+deleted == 0 {
+			return
+		}
+	}
+}
+
+func (p *puller) handleRequestResult(res requestResult) {
+	p.oustandingPerNode.decrease(res.node)
+	f := res.file
+
+	of, ok := p.openFiles[f.Name]
+	if !ok {
+		// no entry in openFiles means there was an error and we've cancelled the operation
+		return
+	}
+
+	if res.err != nil {
+		// This request resulted in an error
+		of.err = res.err
+		if debug {
+			l.Debugf("pull: not writing %q / %q offset %d: %v; (done=%v, outstanding=%d)", p.repoCfg.ID, f.Name, res.offset, res.err, of.done, of.outstanding)
+		}
+	} else if of.err == nil {
+		// This request was sucessfull and nothing has failed previously either
+		_, of.err = of.file.WriteAt(res.data, res.offset)
+		if debug {
+			l.Debugf("pull: wrote %q / %q offset %d len %d outstanding %d done %v", p.repoCfg.ID, f.Name, res.offset, len(res.data), of.outstanding, of.done)
+		}
+	}
+
+	of.outstanding--
+	p.openFiles[f.Name] = of
+
+	if of.done && of.outstanding == 0 {
+		p.closeFile(f)
+	}
+}
+
+// handleBlock fulfills the block request by copying, ignoring or fetching
+// from the network. Returns true if the block was fully handled
+// synchronously, i.e. if the slot can be reused.
+func (p *puller) handleBlock(b bqBlock) bool {
+	f := b.file
+
+	// For directories, making sure they exist is enough.
+	// Deleted directories we mark as handled and delete later.
+	if protocol.IsDirectory(f.Flags) {
+		if !protocol.IsDeleted(f.Flags) {
+			path := filepath.Join(p.repoCfg.Directory, f.Name)
+			_, err := os.Stat(path)
+			if err != nil && os.IsNotExist(err) {
+				if debug {
+					l.Debugf("create dir: %v", f)
+				}
+				err = os.MkdirAll(path, os.FileMode(f.Flags&0777))
+				if err != nil {
+					p.errors++
+					l.Infof("mkdir: error: %q: %v", path, err)
+				}
+			}
+		} else if debug {
+			l.Debugf("ignore delete dir: %v", f)
+		}
+		p.model.updateLocal(p.repoCfg.ID, f)
+		return true
+	}
+
+	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
+		// We are supposed to copy the entire file, and then fetch nothing.
+		// We don't actually need to make the copy.
+		if debug {
+			l.Debugln("taking shortcut:", f)
+		}
+		fp := filepath.Join(p.repoCfg.Directory, f.Name)
+		t := time.Unix(f.Modified, 0)
+		err := os.Chtimes(fp, t, t)
+		if err != nil {
+			l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+		}
+		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
+			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
+			if err != nil {
+				l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+			}
+		}
+
+		events.Default.Log(events.ItemStarted, map[string]string{
+			"repo": p.repoCfg.ID,
+			"item": f.Name,
+		})
+
+		p.model.updateLocal(p.repoCfg.ID, f)
+		return true
+	}
+
+	of, ok := p.openFiles[f.Name]
+	of.done = b.last
+
+	if !ok {
+		if debug {
+			l.Debugf("pull: %q: opening file %q", p.repoCfg.ID, f.Name)
+		}
+
+		events.Default.Log(events.ItemStarted, map[string]string{
+			"repo": p.repoCfg.ID,
+			"item": f.Name,
+		})
+
+		of.availability = p.model.repoFiles[p.repoCfg.ID].Availability(f.Name)
+		of.filepath = filepath.Join(p.repoCfg.Directory, f.Name)
+		of.temp = filepath.Join(p.repoCfg.Directory, defTempNamer.TempName(f.Name))
+
+		dirName := filepath.Dir(of.filepath)
+		info, err := os.Stat(dirName)
+		if err != nil {
+			err = os.MkdirAll(dirName, 0777)
+			if debug && err != nil {
+				l.Debugf("mkdir: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+			}
+		} else {
+			// We need to make sure the directory is writeable so we can create files in it
+			if dirName != p.repoCfg.Directory {
+				err = os.Chmod(dirName, 0777)
+				if debug && err != nil {
+					l.Debugf("make writeable: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+				}
+			}
+			// Change it back after creating the file, to minimize the time window with incorrect permissions
+			defer os.Chmod(dirName, info.Mode())
+		}
+
+		of.file, of.err = os.Create(of.temp)
+		if of.err != nil {
+			p.errors++
+			l.Infof("create: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
+			if !b.last {
+				p.openFiles[f.Name] = of
+			}
+			return true
+		}
+		osutil.HideFile(of.temp)
+	}
+
+	if of.err != nil {
+		// We have already failed this file.
+		if debug {
+			l.Debugf("pull: error: %q / %q has already failed: %v", p.repoCfg.ID, f.Name, of.err)
+		}
+		if b.last {
+			delete(p.openFiles, f.Name)
+		}
+
+		return true
+	}
+
+	p.openFiles[f.Name] = of
+
+	switch {
+	case len(b.copy) > 0:
+		p.handleCopyBlock(b)
+		return true
+
+	case b.block.Size > 0:
+		return p.handleRequestBlock(b)
+
+	default:
+		p.handleEmptyBlock(b)
+		return true
+	}
+}
+
+func (p *puller) handleCopyBlock(b bqBlock) {
+	// We have blocks to copy from the existing file
+	f := b.file
+	of := p.openFiles[f.Name]
+
+	if debug {
+		l.Debugf("pull: copying %d blocks for %q / %q", len(b.copy), p.repoCfg.ID, f.Name)
+	}
+
+	var exfd *os.File
+	exfd, of.err = os.Open(of.filepath)
+	if of.err != nil {
+		p.errors++
+		l.Infof("open: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
+		of.file.Close()
+		of.file = nil
+
+		p.openFiles[f.Name] = of
+		return
+	}
+	defer exfd.Close()
+
+	for _, b := range b.copy {
+		bs := make([]byte, b.Size)
+		_, of.err = exfd.ReadAt(bs, b.Offset)
+		if of.err == nil {
+			_, of.err = of.file.WriteAt(bs, b.Offset)
+		}
+		if of.err != nil {
+			p.errors++
+			l.Infof("write: error: %q / %q: %v", p.repoCfg.ID, f.Name, of.err)
+			exfd.Close()
+			of.file.Close()
+			of.file = nil
+
+			p.openFiles[f.Name] = of
+			return
+		}
+	}
+}
+
+// handleRequestBlock tries to pull a block from the network. Returns true if
+// the block could _not_ be fetched (i.e. it was fully handled, matching the
+// return criteria of handleBlock)
+func (p *puller) handleRequestBlock(b bqBlock) bool {
+	f := b.file
+	of, ok := p.openFiles[f.Name]
+	if !ok {
+		panic("bug: request for non-open file")
+	}
+
+	node := p.oustandingPerNode.leastBusyNode(of.availability, p.model.ConnectedTo)
+	if node == (protocol.NodeID{}) {
+		of.err = errNoNode
+		if of.file != nil {
+			of.file.Close()
+			of.file = nil
+			os.Remove(of.temp)
+			if debug {
+				l.Debugf("pull: no source for %q / %q; closed", p.repoCfg.ID, f.Name)
+			}
+		}
+		if b.last {
+			if debug {
+				l.Debugf("pull: no source for %q / %q; deleting", p.repoCfg.ID, f.Name)
+			}
+			delete(p.openFiles, f.Name)
+		} else {
+			if debug {
+				l.Debugf("pull: no source for %q / %q; await more blocks", p.repoCfg.ID, f.Name)
+			}
+			p.openFiles[f.Name] = of
+		}
+		return true
+	}
+
+	of.outstanding++
+	p.openFiles[f.Name] = of
+
+	go func(node protocol.NodeID, b bqBlock) {
+		if debug {
+			l.Debugf("pull: requesting %q / %q offset %d size %d from %q outstanding %d", p.repoCfg.ID, f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
+		}
+
+		bs, err := p.model.requestGlobal(node, p.repoCfg.ID, f.Name, b.block.Offset, int(b.block.Size), nil)
+		p.requestResults <- requestResult{
+			node:     node,
+			file:     f,
+			filepath: of.filepath,
+			offset:   b.block.Offset,
+			data:     bs,
+			err:      err,
+		}
+	}(node, b)
+
+	return false
+}
+
+func (p *puller) handleEmptyBlock(b bqBlock) {
+	f := b.file
+	of := p.openFiles[f.Name]
+
+	if b.last {
+		if of.err == nil {
+			of.file.Close()
+		}
+	}
+
+	if protocol.IsDeleted(f.Flags) {
+		if debug {
+			l.Debugf("pull: delete %q", f.Name)
+		}
+		os.Remove(of.temp)
+
+		// Ensure the file and the directory it is in is writeable so we can remove the file
+		dirName := filepath.Dir(of.filepath)
+		err := os.Chmod(of.filepath, 0666)
+		if debug && err != nil {
+			l.Debugf("make writeable: error: %q: %v", of.filepath, err)
+		}
+		if dirName != p.repoCfg.Directory {
+			info, err := os.Stat(dirName)
+			if err != nil {
+				l.Debugln("weird! can't happen?", err)
+			}
+			err = os.Chmod(dirName, 0777)
+			if debug && err != nil {
+				l.Debugf("make writeable: error: %q: %v", dirName, err)
+			}
+			// Change it back after deleting the file, to minimize the time window with incorrect permissions
+			defer os.Chmod(dirName, info.Mode())
+		}
+		if p.versioner != nil {
+			if debug {
+				l.Debugln("pull: deleting with versioner")
+			}
+			if err := p.versioner.Archive(of.filepath); err == nil {
+				p.model.updateLocal(p.repoCfg.ID, f)
+			} else if debug {
+				l.Debugln("pull: error:", err)
+			}
+		} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
+			p.model.updateLocal(p.repoCfg.ID, f)
+		}
+	} else {
+		if debug {
+			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repoCfg.ID, f.Name)
+		}
+		t := time.Unix(f.Modified, 0)
+		if os.Chtimes(of.temp, t, t) != nil {
+			delete(p.openFiles, f.Name)
+			return
+		}
+		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) && os.Chmod(of.temp, os.FileMode(f.Flags&0777)) != nil {
+			delete(p.openFiles, f.Name)
+			return
+		}
+		osutil.ShowFile(of.temp)
+		if osutil.Rename(of.temp, of.filepath) == nil {
+			p.model.updateLocal(p.repoCfg.ID, f)
+		}
+	}
+	delete(p.openFiles, f.Name)
+}
+
+func (p *puller) queueNeededBlocks(prevVer uint64) (uint64, int) {
+	curVer := p.model.LocalVersion(p.repoCfg.ID)
+	if curVer == prevVer {
+		return curVer, 0
+	}
+
+	if debug {
+		l.Debugf("%q: checking for more needed blocks", p.repoCfg.ID)
+	}
+
+	queued := 0
+	files := make([]protocol.FileInfo, 0, indexBatchSize)
+	for _, f := range p.model.NeedFilesRepoLimited(p.repoCfg.ID, indexBatchSize, pullIterationBlocks) {
+		if _, ok := p.openFiles[f.Name]; ok {
+			continue
+		}
+		files = append(files, f)
+	}
+
+	perm := rand.Perm(len(files))
+	for _, idx := range perm {
+		f := files[idx]
+		lf := p.model.CurrentRepoFile(p.repoCfg.ID, f.Name)
+		have, need := scanner.BlockDiff(lf.Blocks, f.Blocks)
+		if debug {
+			l.Debugf("need:\n  local: %v\n  global: %v\n  haveBlocks: %v\n  needBlocks: %v", lf, f, have, need)
+		}
+		queued++
+		p.bq.put(bqAdd{
+			file: f,
+			have: have,
+			need: need,
+		})
+	}
+
+	if debug && queued > 0 {
+		l.Debugf("%q: queued %d items", p.repoCfg.ID, queued)
+	}
+
+	if queued > 0 {
+		return prevVer, queued
+	} else {
+		return curVer, 0
+	}
+}
+
+func (p *puller) closeFile(f protocol.FileInfo) {
+	if debug {
+		l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
+	}
+
+	of := p.openFiles[f.Name]
+	err := of.file.Close()
+	if err != nil {
+		p.errors++
+		l.Infof("close: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+	}
+	defer os.Remove(of.temp)
+
+	delete(p.openFiles, f.Name)
+
+	fd, err := os.Open(of.temp)
+	if err != nil {
+		p.errors++
+		l.Infof("open: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+		return
+	}
+	hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize, f.Size())
+	fd.Close()
+
+	if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
+		if debug {
+			l.Debugf("pull: %q / %q: nblocks %d != %d", p.repoCfg.ID, f.Name, l0, l1)
+		}
+		return
+	}
+
+	for i := range hb {
+		if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
+			if debug {
+				l.Debugf("pull: %q / %q: block %d hash mismatch\n  have: %x\n  want: %x", p.repoCfg.ID, f.Name, i, hb[i].Hash, f.Blocks[i].Hash)
+			}
+			return
+		}
+	}
+
+	t := time.Unix(f.Modified, 0)
+	err = os.Chtimes(of.temp, t, t)
+	if err != nil {
+		l.Infof("chtimes: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+	}
+	if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
+		err = os.Chmod(of.temp, os.FileMode(f.Flags&0777))
+		if err != nil {
+			l.Infof("chmod: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+		}
+	}
+
+	osutil.ShowFile(of.temp)
+
+	if p.versioner != nil {
+		err := p.versioner.Archive(of.filepath)
+		if err != nil {
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+			}
+			return
+		}
+	}
+
+	if debug {
+		l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath)
+	}
+	if err := osutil.Rename(of.temp, of.filepath); err == nil {
+		p.model.updateLocal(p.repoCfg.ID, f)
+	} else {
+		p.errors++
+		l.Infof("rename: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+	}
+}
+
+func invalidateRepo(cfg *config.Configuration, repoID string, err error) {
+	for i := range cfg.Repositories {
+		repo := &cfg.Repositories[i]
+		if repo.ID == repoID {
+			repo.Invalid = err.Error()
+			return
+		}
+	}
+}

internal/model/tempname.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !windows
+
+package model
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type tempNamer struct {
+	prefix string
+}
+
+var defTempNamer = tempNamer{".syncthing"}
+
+func (t tempNamer) IsTemporary(name string) bool {
+	return strings.HasPrefix(filepath.Base(name), t.prefix)
+}
+
+func (t tempNamer) TempName(name string) string {
+	tdir := filepath.Dir(name)
+	tname := fmt.Sprintf("%s.%s", t.prefix, filepath.Base(name))
+	return filepath.Join(tdir, tname)
+}

internal/model/tempname_windows.go

@@ -0,0 +1,29 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package model
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type tempNamer struct {
+	prefix string
+}
+
+var defTempNamer = tempNamer{"~syncthing~"}
+
+func (t tempNamer) IsTemporary(name string) bool {
+	return strings.HasPrefix(filepath.Base(name), t.prefix)
+}
+
+func (t tempNamer) TempName(name string) string {
+	tdir := filepath.Dir(name)
+	tname := fmt.Sprintf("%s.%s.tmp", t.prefix, filepath.Base(name))
+	return filepath.Join(tdir, tname)
+}

internal/model/testdata/.stignore

@@ -0,0 +1,2 @@
+.*
+quux

internal/model/testdata/bar

@@ -0,0 +1 @@
+foobarbaz

internal/model/testdata/baz/quux

@@ -0,0 +1 @@
+baazquux

internal/model/testdata/foo

@@ -0,0 +1 @@
+foobar

internal/model/util.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package model
+
+import (
+	"sync"
+	"time"
+)
+
+func deadlockDetect(mut sync.Locker, timeout time.Duration) {
+	go func() {
+		for {
+			time.Sleep(timeout / 4)
+			ok := make(chan bool, 2)
+
+			go func() {
+				mut.Lock()
+				mut.Unlock()
+				ok <- true
+			}()
+
+			go func() {
+				time.Sleep(timeout)
+				ok <- false
+			}()
+
+			if r := <-ok; !r {
+				panic("deadlock detected")
+			}
+		}
+	}()
+}

internal/osutil/hidden_unix.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !windows
+
+package osutil
+
+func HideFile(path string) error {
+	return nil
+}
+
+func ShowFile(path string) error {
+	return nil
+}

internal/osutil/hidden_windows.go

@@ -0,0 +1,39 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package osutil
+
+import "syscall"
+
+func HideFile(path string) error {
+	p, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}
+
+func ShowFile(path string) error {
+	p, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	attrs, err := syscall.GetFileAttributes(p)
+	if err != nil {
+		return err
+	}
+
+	attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
+	return syscall.SetFileAttributes(p, attrs)
+}

internal/osutil/osutil.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package osutil implements utilities for native OS support.
+package osutil
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+)
+
+// Try to keep this entire operation atomic-like. We shouldn't be doing this
+// often enough that there is any contention on this lock.
+var renameLock sync.Mutex
+
+// Rename renames a file, while trying hard to succeed on various systems by
+// temporarily tweaking directory permissions and removing the destination
+// file when necessary. Will make sure to delete the from file if the
+// operation fails, so use only for situations like committing a temp file to
+// it's final location.
+func Rename(from, to string) error {
+	renameLock.Lock()
+	defer renameLock.Unlock()
+
+	// Make sure the destination directory is writeable
+	toDir := filepath.Dir(to)
+	if info, err := os.Stat(toDir); err == nil {
+		os.Chmod(toDir, 0777)
+		defer os.Chmod(toDir, info.Mode())
+	}
+
+	// On Windows, make sure the destination file is writeable (or we can't delete it)
+	if runtime.GOOS == "windows" {
+		os.Chmod(to, 0666)
+		err := os.Remove(to)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	// Don't leave a dangling temp file in case of rename error
+	defer os.Remove(from)
+	return os.Rename(from, to)
+}

internal/osutil/osutil_test.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package osutil_test
+
+// Empty test file to generate 0% coverage rather than no coverage

internal/protocol/.gitignore

@@ -0,0 +1 @@
+*.txt

internal/protocol/common_test.go

@@ -0,0 +1,76 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"io"
+	"time"
+)
+
+type TestModel struct {
+	data     []byte
+	repo     string
+	name     string
+	offset   int64
+	size     int
+	closedCh chan bool
+}
+
+func newTestModel() *TestModel {
+	return &TestModel{
+		closedCh: make(chan bool),
+	}
+}
+
+func (t *TestModel) Index(nodeID NodeID, repo string, files []FileInfo) {
+}
+
+func (t *TestModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) {
+}
+
+func (t *TestModel) Request(nodeID NodeID, repo, name string, offset int64, size int) ([]byte, error) {
+	t.repo = repo
+	t.name = name
+	t.offset = offset
+	t.size = size
+	return t.data, nil
+}
+
+func (t *TestModel) Close(nodeID NodeID, err error) {
+	close(t.closedCh)
+}
+
+func (t *TestModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) {
+}
+
+func (t *TestModel) isClosed() bool {
+	select {
+	case <-t.closedCh:
+		return true
+	case <-time.After(1 * time.Second):
+		return false // Timeout
+	}
+}
+
+type ErrPipe struct {
+	io.PipeWriter
+	written int
+	max     int
+	err     error
+	closed  bool
+}
+
+func (e *ErrPipe) Write(data []byte) (int, error) {
+	if e.closed {
+		return 0, e.err
+	}
+	if e.written+len(data) > e.max {
+		n, _ := e.PipeWriter.Write(data[:e.max-e.written])
+		e.PipeWriter.CloseWithError(e.err)
+		e.closed = true
+		return n, e.err
+	}
+	return e.PipeWriter.Write(data)
+}

internal/protocol/counting.go

@@ -0,0 +1,64 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"io"
+	"sync/atomic"
+	"time"
+)
+
+type countingReader struct {
+	io.Reader
+	tot  uint64 // bytes
+	last int64  // unix nanos
+}
+
+var (
+	totalIncoming uint64
+	totalOutgoing uint64
+)
+
+func (c *countingReader) Read(bs []byte) (int, error) {
+	n, err := c.Reader.Read(bs)
+	atomic.AddUint64(&c.tot, uint64(n))
+	atomic.AddUint64(&totalIncoming, uint64(n))
+	atomic.StoreInt64(&c.last, time.Now().UnixNano())
+	return n, err
+}
+
+func (c *countingReader) Tot() uint64 {
+	return atomic.LoadUint64(&c.tot)
+}
+
+func (c *countingReader) Last() time.Time {
+	return time.Unix(0, atomic.LoadInt64(&c.last))
+}
+
+type countingWriter struct {
+	io.Writer
+	tot  uint64 // bytes
+	last int64  // unix nanos
+}
+
+func (c *countingWriter) Write(bs []byte) (int, error) {
+	n, err := c.Writer.Write(bs)
+	atomic.AddUint64(&c.tot, uint64(n))
+	atomic.AddUint64(&totalOutgoing, uint64(n))
+	atomic.StoreInt64(&c.last, time.Now().UnixNano())
+	return n, err
+}
+
+func (c *countingWriter) Tot() uint64 {
+	return atomic.LoadUint64(&c.tot)
+}
+
+func (c *countingWriter) Last() time.Time {
+	return time.Unix(0, atomic.LoadInt64(&c.last))
+}
+
+func TotalInOut() (uint64, uint64) {
+	return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing)
+}

internal/protocol/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/protocol/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package protocol implements the Block Exchange Protocol.
+package protocol

internal/protocol/header.go

@@ -0,0 +1,45 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import "github.com/calmh/xdr"
+
+type header struct {
+	version     int
+	msgID       int
+	msgType     int
+	compression bool
+}
+
+func (h header) encodeXDR(xw *xdr.Writer) (int, error) {
+	u := encodeHeader(h)
+	return xw.WriteUint32(u)
+}
+
+func (h *header) decodeXDR(xr *xdr.Reader) error {
+	u := xr.ReadUint32()
+	*h = decodeHeader(u)
+	return xr.Error()
+}
+
+func encodeHeader(h header) uint32 {
+	var isComp uint32
+	if h.compression {
+		isComp = 1 << 0 // the zeroth bit is the compression bit
+	}
+	return uint32(h.version&0xf)<<28 +
+		uint32(h.msgID&0xfff)<<16 +
+		uint32(h.msgType&0xff)<<8 +
+		isComp
+}
+
+func decodeHeader(u uint32) header {
+	return header{
+		version:     int(u>>28) & 0xf,
+		msgID:       int(u>>16) & 0xfff,
+		msgType:     int(u>>8) & 0xff,
+		compression: u&1 == 1,
+	}
+}

internal/protocol/message.go

@@ -0,0 +1,139 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import "fmt"
+
+type IndexMessage struct {
+	Repository string // max:64
+	Files      []FileInfo
+}
+
+type FileInfo struct {
+	Name         string // max:8192
+	Flags        uint32
+	Modified     int64
+	Version      uint64
+	LocalVersion uint64
+	Blocks       []BlockInfo
+}
+
+func (f FileInfo) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
+}
+
+func (f FileInfo) Size() (bytes int64) {
+	if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
+		return 128
+	}
+	for _, b := range f.Blocks {
+		bytes += int64(b.Size)
+	}
+	return
+}
+
+func (f FileInfo) IsDeleted() bool {
+	return IsDeleted(f.Flags)
+}
+
+func (f FileInfo) IsInvalid() bool {
+	return IsInvalid(f.Flags)
+}
+
+// Used for unmarshalling a FileInfo structure but skipping the actual block list
+type FileInfoTruncated struct {
+	Name         string // max:8192
+	Flags        uint32
+	Modified     int64
+	Version      uint64
+	LocalVersion uint64
+	NumBlocks    uint32
+}
+
+// Returns a statistical guess on the size, not the exact figure
+func (f FileInfoTruncated) Size() int64 {
+	if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
+		return 128
+	}
+	if f.NumBlocks < 2 {
+		return BlockSize / 2
+	} else {
+		return int64(f.NumBlocks-1)*BlockSize + BlockSize/2
+	}
+}
+
+func (f FileInfoTruncated) IsDeleted() bool {
+	return IsDeleted(f.Flags)
+}
+
+func (f FileInfoTruncated) IsInvalid() bool {
+	return IsInvalid(f.Flags)
+}
+
+type FileIntf interface {
+	Size() int64
+	IsDeleted() bool
+	IsInvalid() bool
+}
+
+type BlockInfo struct {
+	Offset int64 // noencode (cache only)
+	Size   uint32
+	Hash   []byte // max:64
+}
+
+func (b BlockInfo) String() string {
+	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
+}
+
+type RequestMessage struct {
+	Repository string // max:64
+	Name       string // max:8192
+	Offset     uint64
+	Size       uint32
+}
+
+type ResponseMessage struct {
+	Data []byte
+}
+
+type ClusterConfigMessage struct {
+	ClientName    string       // max:64
+	ClientVersion string       // max:64
+	Repositories  []Repository // max:64
+	Options       []Option     // max:64
+}
+
+func (o *ClusterConfigMessage) GetOption(key string) string {
+	for _, option := range o.Options {
+		if option.Key == key {
+			return option.Value
+		}
+	}
+	return ""
+}
+
+type Repository struct {
+	ID    string // max:64
+	Nodes []Node // max:64
+}
+
+type Node struct {
+	ID              []byte // max:32
+	Flags           uint32
+	MaxLocalVersion uint64
+}
+
+type Option struct {
+	Key   string // max:64
+	Value string // max:1024
+}
+
+type CloseMessage struct {
+	Reason string // max:1024
+}
+
+type EmptyMessage struct{}

internal/protocol/message_xdr.go

@@ -0,0 +1,964 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package protocol
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+IndexMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                     Length of Repository                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Repository (variable length)                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Number of Files                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\               Zero or more FileInfo Structures                \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct IndexMessage {
+	string Repository<64>;
+	FileInfo Files<>;
+}
+
+*/
+
+func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o IndexMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o IndexMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Repository) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Repository)
+	xw.WriteUint32(uint32(len(o.Files)))
+	for i := range o.Files {
+		_, err := o.Files[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *IndexMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *IndexMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
+	o.Repository = xr.ReadStringMax(64)
+	_FilesSize := int(xr.ReadUint32())
+	o.Files = make([]FileInfo, _FilesSize)
+	for i := range o.Files {
+		(&o.Files[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+FileInfo Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Name                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Name (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                      Modified (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                       Version (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                    Local Version (64 bits)                    +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Blocks                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\               Zero or more BlockInfo Structures               \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct FileInfo {
+	string Name<8192>;
+	unsigned int Flags;
+	hyper Modified;
+	unsigned hyper Version;
+	unsigned hyper LocalVersion;
+	BlockInfo Blocks<>;
+}
+
+*/
+
+func (o FileInfo) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o FileInfo) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o FileInfo) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Name) > 8192 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Name)
+	xw.WriteUint32(o.Flags)
+	xw.WriteUint64(uint64(o.Modified))
+	xw.WriteUint64(o.Version)
+	xw.WriteUint64(o.LocalVersion)
+	xw.WriteUint32(uint32(len(o.Blocks)))
+	for i := range o.Blocks {
+		_, err := o.Blocks[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *FileInfo) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfo) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
+	o.Name = xr.ReadStringMax(8192)
+	o.Flags = xr.ReadUint32()
+	o.Modified = int64(xr.ReadUint64())
+	o.Version = xr.ReadUint64()
+	o.LocalVersion = xr.ReadUint64()
+	_BlocksSize := int(xr.ReadUint32())
+	o.Blocks = make([]BlockInfo, _BlocksSize)
+	for i := range o.Blocks {
+		(&o.Blocks[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+FileInfoTruncated Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Name                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Name (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                      Modified (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                       Version (64 bits)                       +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                    Local Version (64 bits)                    +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                          Num Blocks                           |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct FileInfoTruncated {
+	string Name<8192>;
+	unsigned int Flags;
+	hyper Modified;
+	unsigned hyper Version;
+	unsigned hyper LocalVersion;
+	unsigned int NumBlocks;
+}
+
+*/
+
+func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o FileInfoTruncated) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o FileInfoTruncated) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Name) > 8192 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Name)
+	xw.WriteUint32(o.Flags)
+	xw.WriteUint64(uint64(o.Modified))
+	xw.WriteUint64(o.Version)
+	xw.WriteUint64(o.LocalVersion)
+	xw.WriteUint32(o.NumBlocks)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error {
+	o.Name = xr.ReadStringMax(8192)
+	o.Flags = xr.ReadUint32()
+	o.Modified = int64(xr.ReadUint64())
+	o.Version = xr.ReadUint64()
+	o.LocalVersion = xr.ReadUint64()
+	o.NumBlocks = xr.ReadUint32()
+	return xr.Error()
+}
+
+/*
+
+BlockInfo Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Size                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Hash                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Hash (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct BlockInfo {
+	unsigned int Size;
+	opaque Hash<64>;
+}
+
+*/
+
+func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o BlockInfo) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o BlockInfo) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint32(o.Size)
+	if len(o.Hash) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.Hash)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *BlockInfo) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *BlockInfo) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error {
+	o.Size = xr.ReadUint32()
+	o.Hash = xr.ReadBytesMax(64)
+	return xr.Error()
+}
+
+/*
+
+RequestMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                     Length of Repository                      |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Repository (variable length)                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Name                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Name (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                       Offset (64 bits)                        +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Size                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct RequestMessage {
+	string Repository<64>;
+	string Name<8192>;
+	unsigned hyper Offset;
+	unsigned int Size;
+}
+
+*/
+
+func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o RequestMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o RequestMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Repository) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Repository)
+	if len(o.Name) > 8192 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Name)
+	xw.WriteUint64(o.Offset)
+	xw.WriteUint32(o.Size)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *RequestMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *RequestMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
+	o.Repository = xr.ReadStringMax(64)
+	o.Name = xr.ReadStringMax(8192)
+	o.Offset = xr.ReadUint64()
+	o.Size = xr.ReadUint32()
+	return xr.Error()
+}
+
+/*
+
+ResponseMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Data                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Data (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct ResponseMessage {
+	opaque Data<>;
+}
+
+*/
+
+func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o ResponseMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o ResponseMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteBytes(o.Data)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *ResponseMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *ResponseMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error {
+	o.Data = xr.ReadBytes()
+	return xr.Error()
+}
+
+/*
+
+ClusterConfigMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                     Length of Client Name                     |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Client Name (variable length)                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                   Length of Client Version                    |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\               Client Version (variable length)                \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                    Number of Repositories                     |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\              Zero or more Repository Structures               \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Options                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Option Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct ClusterConfigMessage {
+	string ClientName<64>;
+	string ClientVersion<64>;
+	Repository Repositories<64>;
+	Option Options<64>;
+}
+
+*/
+
+func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o ClusterConfigMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o ClusterConfigMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ClientName) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ClientName)
+	if len(o.ClientVersion) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ClientVersion)
+	if len(o.Repositories) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Repositories)))
+	for i := range o.Repositories {
+		_, err := o.Repositories[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	if len(o.Options) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Options)))
+	for i := range o.Options {
+		_, err := o.Options[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error {
+	o.ClientName = xr.ReadStringMax(64)
+	o.ClientVersion = xr.ReadStringMax(64)
+	_RepositoriesSize := int(xr.ReadUint32())
+	if _RepositoriesSize > 64 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Repositories = make([]Repository, _RepositoriesSize)
+	for i := range o.Repositories {
+		(&o.Repositories[i]).decodeXDR(xr)
+	}
+	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize > 64 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Options = make([]Option, _OptionsSize)
+	for i := range o.Options {
+		(&o.Options[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Repository Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of ID                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     ID (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Number of Nodes                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                 Zero or more Node Structures                  \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Repository {
+	string ID<64>;
+	Node Nodes<64>;
+}
+
+*/
+
+func (o Repository) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Repository) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Repository) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ID)
+	if len(o.Nodes) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Nodes)))
+	for i := range o.Nodes {
+		_, err := o.Nodes[i].encodeXDR(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Repository) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Repository) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Repository) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadStringMax(64)
+	_NodesSize := int(xr.ReadUint32())
+	if _NodesSize > 64 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Nodes = make([]Node, _NodesSize)
+	for i := range o.Nodes {
+		(&o.Nodes[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+/*
+
+Node Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of ID                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     ID (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                  Max Local Version (64 bits)                  +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Node {
+	opaque ID<32>;
+	unsigned int Flags;
+	unsigned hyper MaxLocalVersion;
+}
+
+*/
+
+func (o Node) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Node) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Node) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 32 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.ID)
+	xw.WriteUint32(o.Flags)
+	xw.WriteUint64(o.MaxLocalVersion)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Node) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadBytesMax(32)
+	o.Flags = xr.ReadUint32()
+	o.MaxLocalVersion = xr.ReadUint64()
+	return xr.Error()
+}
+
+/*
+
+Option Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Key                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Key (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of Value                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    Value (variable length)                    \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct Option {
+	string Key<64>;
+	string Value<1024>;
+}
+
+*/
+
+func (o Option) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Option) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o Option) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o Option) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Key) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Key)
+	if len(o.Value) > 1024 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Value)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Option) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Option) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *Option) decodeXDR(xr *xdr.Reader) error {
+	o.Key = xr.ReadStringMax(64)
+	o.Value = xr.ReadStringMax(1024)
+	return xr.Error()
+}
+
+/*
+
+CloseMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Length of Reason                        |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                   Reason (variable length)                    \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct CloseMessage {
+	string Reason<1024>;
+}
+
+*/
+
+func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o CloseMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o CloseMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.Reason) > 1024 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.Reason)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *CloseMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *CloseMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error {
+	o.Reason = xr.ReadStringMax(1024)
+	return xr.Error()
+}
+
+/*
+
+EmptyMessage Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct EmptyMessage {
+}
+
+*/
+
+func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o EmptyMessage) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o EmptyMessage) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	return xw.Tot(), xw.Error()
+}
+
+func (o *EmptyMessage) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *EmptyMessage) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error {
+	return xr.Error()
+}

internal/protocol/nativemodel_darwin.go

@@ -0,0 +1,42 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build darwin
+
+package protocol
+
+// Darwin uses NFD normalization
+
+import "code.google.com/p/go.text/unicode/norm"
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) {
+	for i := range files {
+		files[i].Name = norm.NFD.String(files[i].Name)
+	}
+	m.next.Index(nodeID, repo, files)
+}
+
+func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) {
+	for i := range files {
+		files[i].Name = norm.NFD.String(files[i].Name)
+	}
+	m.next.IndexUpdate(nodeID, repo, files)
+}
+
+func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) {
+	name = norm.NFD.String(name)
+	return m.next.Request(nodeID, repo, name, offset, size)
+}
+
+func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
+func (m nativeModel) Close(nodeID NodeID, err error) {
+	m.next.Close(nodeID, err)
+}

internal/protocol/nativemodel_unix.go

@@ -0,0 +1,33 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !windows,!darwin
+
+package protocol
+
+// Normal Unixes uses NFC and slashes, which is the wire format.
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) {
+	m.next.Index(nodeID, repo, files)
+}
+
+func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) {
+	m.next.IndexUpdate(nodeID, repo, files)
+}
+
+func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) {
+	return m.next.Request(nodeID, repo, name, offset, size)
+}
+
+func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
+func (m nativeModel) Close(nodeID NodeID, err error) {
+	m.next.Close(nodeID, err)
+}

internal/protocol/nativemodel_windows.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build windows
+
+package protocol
+
+// Windows uses backslashes as file separator and disallows a bunch of
+// characters in the filename
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+var disallowedCharacters = string([]rune{
+	'<', '>', ':', '"', '|', '?', '*',
+	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+	11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+	21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+	31,
+})
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) {
+	for i, f := range files {
+		if strings.ContainsAny(f.Name, disallowedCharacters) {
+			if f.IsDeleted() {
+				// Don't complain if the file is marked as deleted, since it
+				// can't possibly exist here anyway.
+				continue
+			}
+			files[i].Flags |= FlagInvalid
+			l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
+		}
+		files[i].Name = filepath.FromSlash(f.Name)
+	}
+	m.next.Index(nodeID, repo, files)
+}
+
+func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) {
+	for i, f := range files {
+		if strings.ContainsAny(f.Name, disallowedCharacters) {
+			if f.IsDeleted() {
+				// Don't complain if the file is marked as deleted, since it
+				// can't possibly exist here anyway.
+				continue
+			}
+			files[i].Flags |= FlagInvalid
+			l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
+		}
+		files[i].Name = filepath.FromSlash(files[i].Name)
+	}
+	m.next.IndexUpdate(nodeID, repo, files)
+}
+
+func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) {
+	name = filepath.FromSlash(name)
+	return m.next.Request(nodeID, repo, name, offset, size)
+}
+
+func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
+func (m nativeModel) Close(nodeID NodeID, err error) {
+	m.next.Close(nodeID, err)
+}

internal/protocol/nodeid.go

@@ -0,0 +1,159 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base32"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/luhn"
+)
+
+type NodeID [32]byte
+
+var LocalNodeID = NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+
+// NewNodeID generates a new node ID from the raw bytes of a certificate
+func NewNodeID(rawCert []byte) NodeID {
+	var n NodeID
+	hf := sha256.New()
+	hf.Write(rawCert)
+	hf.Sum(n[:0])
+	return n
+}
+
+func NodeIDFromString(s string) (NodeID, error) {
+	var n NodeID
+	err := n.UnmarshalText([]byte(s))
+	return n, err
+}
+
+func NodeIDFromBytes(bs []byte) NodeID {
+	var n NodeID
+	if len(bs) != len(n) {
+		panic("incorrect length of byte slice representing node ID")
+	}
+	copy(n[:], bs)
+	return n
+}
+
+// String returns the canonical string representation of the node ID
+func (n NodeID) String() string {
+	id := base32.StdEncoding.EncodeToString(n[:])
+	id = strings.Trim(id, "=")
+	id, err := luhnify(id)
+	if err != nil {
+		// Should never happen
+		panic(err)
+	}
+	id = chunkify(id)
+	return id
+}
+
+func (n NodeID) GoString() string {
+	return n.String()
+}
+
+func (n NodeID) Compare(other NodeID) int {
+	return bytes.Compare(n[:], other[:])
+}
+
+func (n NodeID) Equals(other NodeID) bool {
+	return bytes.Compare(n[:], other[:]) == 0
+}
+
+func (n *NodeID) MarshalText() ([]byte, error) {
+	return []byte(n.String()), nil
+}
+
+func (n *NodeID) UnmarshalText(bs []byte) error {
+	id := string(bs)
+	id = strings.Trim(id, "=")
+	id = strings.ToUpper(id)
+	id = untypeoify(id)
+	id = unchunkify(id)
+
+	var err error
+	switch len(id) {
+	case 56:
+		// New style, with check digits
+		id, err = unluhnify(id)
+		if err != nil {
+			return err
+		}
+		fallthrough
+	case 52:
+		// Old style, no check digits
+		dec, err := base32.StdEncoding.DecodeString(id + "====")
+		if err != nil {
+			return err
+		}
+		copy(n[:], dec)
+		return nil
+	default:
+		return errors.New("node ID invalid: incorrect length")
+	}
+}
+
+func luhnify(s string) (string, error) {
+	if len(s) != 52 {
+		panic("unsupported string length")
+	}
+
+	res := make([]string, 0, 4)
+	for i := 0; i < 4; i++ {
+		p := s[i*13 : (i+1)*13]
+		l, err := luhn.Base32.Generate(p)
+		if err != nil {
+			return "", err
+		}
+		res = append(res, fmt.Sprintf("%s%c", p, l))
+	}
+	return res[0] + res[1] + res[2] + res[3], nil
+}
+
+func unluhnify(s string) (string, error) {
+	if len(s) != 56 {
+		return "", fmt.Errorf("unsupported string length %d", len(s))
+	}
+
+	res := make([]string, 0, 4)
+	for i := 0; i < 4; i++ {
+		p := s[i*14 : (i+1)*14-1]
+		l, err := luhn.Base32.Generate(p)
+		if err != nil {
+			return "", err
+		}
+		if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] {
+			return "", errors.New("check digit incorrect")
+		}
+		res = append(res, p)
+	}
+	return res[0] + res[1] + res[2] + res[3], nil
+}
+
+func chunkify(s string) string {
+	s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-")
+	s = strings.Trim(s, "-")
+	return s
+}
+
+func unchunkify(s string) string {
+	s = strings.Replace(s, "-", "", -1)
+	s = strings.Replace(s, " ", "", -1)
+	return s
+}
+
+func untypeoify(s string) string {
+	s = strings.Replace(s, "0", "O", -1)
+	s = strings.Replace(s, "1", "I", -1)
+	s = strings.Replace(s, "8", "B", -1)
+	return s
+}

internal/protocol/nodeid_test.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import "testing"
+
+var formatted = "P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"
+var formatCases = []string{
+	"P56IOI-7MZJNU-2IQGDR-EYDM2M-GTMGL3-BXNPQ6-W5BTBB-Z4TJXZ-WICQ",
+	"P56IOI-7MZJNU2Y-IQGDR-EYDM2M-GTI-MGL3-BXNPQ6-W5BM-TBB-Z4TJXZ-WICQ2",
+	"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ",
+	"P56IOI7 MZJNU2Y IQGDREY DM2MGTI MGL3BXN PQ6W5BM TBBZ4TJ XZWICQ2",
+	"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ",
+	"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq",
+	"P56IOI7MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMTBBZ4TJXZWICQ2",
+	"P561017MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMT88Z4TJXZWICQ2",
+	"p56ioi7mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmtbbz4tjxzwicq2",
+	"p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2",
+}
+
+func TestFormatNodeID(t *testing.T) {
+	for i, tc := range formatCases {
+		var id NodeID
+		err := id.UnmarshalText([]byte(tc))
+		if err != nil {
+			t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err)
+		} else if f := id.String(); f != formatted {
+			t.Errorf("#%d FormatNodeID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted)
+		}
+	}
+}
+
+var validateCases = []struct {
+	s  string
+	ok bool
+}{
+	{"", false},
+	{"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true},
+	{"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true},
+	{"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true},
+	{"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", true},
+	{"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQCCCC", false},
+	{"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", true},
+	{"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false},
+}
+
+func TestValidateNodeID(t *testing.T) {
+	for _, tc := range validateCases {
+		var id NodeID
+		err := id.UnmarshalText([]byte(tc.s))
+		if (err == nil && !tc.ok) || (err != nil && tc.ok) {
+			t.Errorf("ValidateNodeID(%q); %v != %v", tc.s, err, tc.ok)
+		}
+	}
+}
+
+func TestMarshallingNodeID(t *testing.T) {
+	n0 := NodeID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}
+	n1 := NodeID{}
+	n2 := NodeID{}
+
+	bs, _ := n0.MarshalText()
+	n1.UnmarshalText(bs)
+	bs, _ = n1.MarshalText()
+	n2.UnmarshalText(bs)
+
+	if n2.String() != n0.String() {
+		t.Errorf("String marshalling error; %q != %q", n2.String(), n0.String())
+	}
+	if !n2.Equals(n0) {
+		t.Error("Equals error")
+	}
+	if n2.Compare(n0) != 0 {
+		t.Error("Compare error")
+	}
+}

internal/protocol/protocol.go

@@ -0,0 +1,640 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	lz4 "github.com/bkaradzic/go-lz4"
+)
+
+const (
+	BlockSize = 128 * 1024
+)
+
+const (
+	messageTypeClusterConfig = 0
+	messageTypeIndex         = 1
+	messageTypeRequest       = 2
+	messageTypeResponse      = 3
+	messageTypePing          = 4
+	messageTypePong          = 5
+	messageTypeIndexUpdate   = 6
+	messageTypeClose         = 7
+)
+
+const (
+	stateInitial = iota
+	stateCCRcvd
+	stateIdxRcvd
+)
+
+const (
+	FlagDeleted    uint32 = 1 << 12
+	FlagInvalid           = 1 << 13
+	FlagDirectory         = 1 << 14
+	FlagNoPermBits        = 1 << 15
+)
+
+const (
+	FlagShareTrusted  uint32 = 1 << 0
+	FlagShareReadOnly        = 1 << 1
+	FlagIntroducer           = 1 << 2
+	FlagShareBits            = 0x000000ff
+)
+
+var (
+	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
+	ErrClosed      = errors.New("connection closed")
+)
+
+type Model interface {
+	// An index was received from the peer node
+	Index(nodeID NodeID, repo string, files []FileInfo)
+	// An index update was received from the peer node
+	IndexUpdate(nodeID NodeID, repo string, files []FileInfo)
+	// A request was made by the peer node
+	Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error)
+	// A cluster configuration message was received
+	ClusterConfig(nodeID NodeID, config ClusterConfigMessage)
+	// The peer node closed the connection
+	Close(nodeID NodeID, err error)
+}
+
+type Connection interface {
+	ID() NodeID
+	Name() string
+	Index(repo string, files []FileInfo) error
+	IndexUpdate(repo string, files []FileInfo) error
+	Request(repo string, name string, offset int64, size int) ([]byte, error)
+	ClusterConfig(config ClusterConfigMessage)
+	Statistics() Statistics
+}
+
+type rawConnection struct {
+	id       NodeID
+	name     string
+	receiver Model
+	state    int
+
+	cr *countingReader
+	cw *countingWriter
+
+	awaiting    [4096]chan asyncResult
+	awaitingMut sync.Mutex
+
+	idxMut sync.Mutex // ensures serialization of Index calls
+
+	nextID chan int
+	outbox chan hdrMsg
+	closed chan struct{}
+	once   sync.Once
+
+	compressionThreshold int // compress messages larger than this many bytes
+
+	rdbuf0 []byte // used & reused by readMessage
+	rdbuf1 []byte // used & reused by readMessage
+}
+
+type asyncResult struct {
+	val []byte
+	err error
+}
+
+type hdrMsg struct {
+	hdr header
+	msg encodable
+}
+
+type encodable interface {
+	AppendXDR([]byte) []byte
+}
+
+const (
+	pingTimeout  = 30 * time.Second
+	pingIdleTime = 60 * time.Second
+)
+
+func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection {
+	cr := &countingReader{Reader: reader}
+	cw := &countingWriter{Writer: writer}
+
+	compThres := 1<<31 - 1 // compression disabled
+	if compress {
+		compThres = 128 // compress messages that are 128 bytes long or larger
+	}
+	c := rawConnection{
+		id:                   nodeID,
+		name:                 name,
+		receiver:             nativeModel{receiver},
+		state:                stateInitial,
+		cr:                   cr,
+		cw:                   cw,
+		outbox:               make(chan hdrMsg),
+		nextID:               make(chan int),
+		closed:               make(chan struct{}),
+		compressionThreshold: compThres,
+	}
+
+	go c.readerLoop()
+	go c.writerLoop()
+	go c.pingerLoop()
+	go c.idGenerator()
+
+	return wireFormatConnection{&c}
+}
+
+func (c *rawConnection) ID() NodeID {
+	return c.id
+}
+
+func (c *rawConnection) Name() string {
+	return c.name
+}
+
+// Index writes the list of file information to the connected peer node
+func (c *rawConnection) Index(repo string, idx []FileInfo) error {
+	select {
+	case <-c.closed:
+		return ErrClosed
+	default:
+	}
+	c.idxMut.Lock()
+	c.send(-1, messageTypeIndex, IndexMessage{repo, idx})
+	c.idxMut.Unlock()
+	return nil
+}
+
+// IndexUpdate writes the list of file information to the connected peer node as an update
+func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error {
+	select {
+	case <-c.closed:
+		return ErrClosed
+	default:
+	}
+	c.idxMut.Lock()
+	c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx})
+	c.idxMut.Unlock()
+	return nil
+}
+
+// Request returns the bytes for the specified block after fetching them from the connected peer.
+func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
+		return nil, ErrClosed
+	}
+
+	c.awaitingMut.Lock()
+	if ch := c.awaiting[id]; ch != nil {
+		panic("id taken")
+	}
+	rc := make(chan asyncResult, 1)
+	c.awaiting[id] = rc
+	c.awaitingMut.Unlock()
+
+	ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)})
+	if !ok {
+		return nil, ErrClosed
+	}
+
+	res, ok := <-rc
+	if !ok {
+		return nil, ErrClosed
+	}
+	return res.val, res.err
+}
+
+// ClusterConfig send the cluster configuration message to the peer and returns any error
+func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
+	c.send(-1, messageTypeClusterConfig, config)
+}
+
+func (c *rawConnection) ping() bool {
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
+		return false
+	}
+
+	rc := make(chan asyncResult, 1)
+	c.awaitingMut.Lock()
+	c.awaiting[id] = rc
+	c.awaitingMut.Unlock()
+
+	ok := c.send(id, messageTypePing, nil)
+	if !ok {
+		return false
+	}
+
+	res, ok := <-rc
+	return ok && res.err == nil
+}
+
+func (c *rawConnection) readerLoop() (err error) {
+	defer func() {
+		c.close(err)
+	}()
+
+	for {
+		select {
+		case <-c.closed:
+			return ErrClosed
+		default:
+		}
+
+		hdr, msg, err := c.readMessage()
+		if err != nil {
+			return err
+		}
+
+		switch hdr.msgType {
+		case messageTypeIndex:
+			if c.state < stateCCRcvd {
+				return fmt.Errorf("protocol error: index message in state %d", c.state)
+			}
+			c.handleIndex(msg.(IndexMessage))
+			c.state = stateIdxRcvd
+
+		case messageTypeIndexUpdate:
+			if c.state < stateIdxRcvd {
+				return fmt.Errorf("protocol error: index update message in state %d", c.state)
+			}
+			c.handleIndexUpdate(msg.(IndexMessage))
+
+		case messageTypeRequest:
+			if c.state < stateIdxRcvd {
+				return fmt.Errorf("protocol error: request message in state %d", c.state)
+			}
+			// Requests are handled asynchronously
+			go c.handleRequest(hdr.msgID, msg.(RequestMessage))
+
+		case messageTypeResponse:
+			if c.state < stateIdxRcvd {
+				return fmt.Errorf("protocol error: response message in state %d", c.state)
+			}
+			c.handleResponse(hdr.msgID, msg.(ResponseMessage))
+
+		case messageTypePing:
+			c.send(hdr.msgID, messageTypePong, EmptyMessage{})
+
+		case messageTypePong:
+			c.handlePong(hdr.msgID)
+
+		case messageTypeClusterConfig:
+			if c.state != stateInitial {
+				return fmt.Errorf("protocol error: cluster config message in state %d", c.state)
+			}
+			go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage))
+			c.state = stateCCRcvd
+
+		case messageTypeClose:
+			return errors.New(msg.(CloseMessage).Reason)
+
+		default:
+			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+		}
+	}
+}
+
+func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
+	if cap(c.rdbuf0) < 8 {
+		c.rdbuf0 = make([]byte, 8)
+	} else {
+		c.rdbuf0 = c.rdbuf0[:8]
+	}
+	_, err = io.ReadFull(c.cr, c.rdbuf0)
+	if err != nil {
+		return
+	}
+
+	hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4]))
+	msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8]))
+
+	if debug {
+		l.Debugf("read header %v (msglen=%d)", hdr, msglen)
+	}
+
+	if cap(c.rdbuf0) < msglen {
+		c.rdbuf0 = make([]byte, msglen)
+	} else {
+		c.rdbuf0 = c.rdbuf0[:msglen]
+	}
+	_, err = io.ReadFull(c.cr, c.rdbuf0)
+	if err != nil {
+		return
+	}
+
+	if debug {
+		l.Debugf("read %d bytes", len(c.rdbuf0))
+	}
+
+	msgBuf := c.rdbuf0
+	if hdr.compression {
+		c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)]
+		c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0)
+		if err != nil {
+			return
+		}
+		msgBuf = c.rdbuf1
+		if debug {
+			l.Debugf("decompressed to %d bytes", len(msgBuf))
+		}
+	}
+
+	if debug {
+		if len(msgBuf) > 1024 {
+			l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024]))
+		} else {
+			l.Debugf("message data:\n%s", hex.Dump(msgBuf))
+		}
+	}
+
+	switch hdr.msgType {
+	case messageTypeIndex, messageTypeIndexUpdate:
+		var idx IndexMessage
+		err = idx.UnmarshalXDR(msgBuf)
+		msg = idx
+
+	case messageTypeRequest:
+		var req RequestMessage
+		err = req.UnmarshalXDR(msgBuf)
+		msg = req
+
+	case messageTypeResponse:
+		var resp ResponseMessage
+		err = resp.UnmarshalXDR(msgBuf)
+		msg = resp
+
+	case messageTypePing, messageTypePong:
+		msg = EmptyMessage{}
+
+	case messageTypeClusterConfig:
+		var cc ClusterConfigMessage
+		err = cc.UnmarshalXDR(msgBuf)
+		msg = cc
+
+	case messageTypeClose:
+		var cm CloseMessage
+		err = cm.UnmarshalXDR(msgBuf)
+		msg = cm
+
+	default:
+		err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+	}
+
+	return
+}
+
+func (c *rawConnection) handleIndex(im IndexMessage) {
+	if debug {
+		l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
+	}
+	c.receiver.Index(c.id, im.Repository, im.Files)
+}
+
+func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
+	if debug {
+		l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
+	}
+	c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
+}
+
+func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
+	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
+
+	c.send(msgID, messageTypeResponse, ResponseMessage{data})
+}
+
+func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
+	c.awaitingMut.Lock()
+	if rc := c.awaiting[msgID]; rc != nil {
+		c.awaiting[msgID] = nil
+		rc <- asyncResult{resp.Data, nil}
+		close(rc)
+	}
+	c.awaitingMut.Unlock()
+}
+
+func (c *rawConnection) handlePong(msgID int) {
+	c.awaitingMut.Lock()
+	if rc := c.awaiting[msgID]; rc != nil {
+		c.awaiting[msgID] = nil
+		rc <- asyncResult{}
+		close(rc)
+	}
+	c.awaitingMut.Unlock()
+}
+
+func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool {
+	if msgID < 0 {
+		select {
+		case id := <-c.nextID:
+			msgID = id
+		case <-c.closed:
+			return false
+		}
+	}
+
+	hdr := header{
+		version: 0,
+		msgID:   msgID,
+		msgType: msgType,
+	}
+
+	select {
+	case c.outbox <- hdrMsg{hdr, msg}:
+		return true
+	case <-c.closed:
+		return false
+	}
+}
+
+func (c *rawConnection) writerLoop() {
+	var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
+	var uncBuf []byte            // buffer for uncompressed message, kept and reused
+	for {
+		var tempBuf []byte
+		var err error
+
+		select {
+		case hm := <-c.outbox:
+			if hm.msg != nil {
+				// Uncompressed message in uncBuf
+				uncBuf = hm.msg.AppendXDR(uncBuf[:0])
+
+				if len(uncBuf) >= c.compressionThreshold {
+					// Use compression for large messages
+					hm.hdr.compression = true
+
+					// Make sure we have enough space for the compressed message plus header in msgBug
+					msgBuf = msgBuf[:cap(msgBuf)]
+					if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
+						msgBuf = make([]byte, maxLen)
+					}
+
+					// Compressed is written to msgBuf, we keep tb for the length only
+					tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
+					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
+					msgBuf = msgBuf[0 : len(tempBuf)+8]
+
+					if debug {
+						l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
+					}
+				} else {
+					// No point in compressing very short messages
+					hm.hdr.compression = false
+
+					msgBuf = msgBuf[:cap(msgBuf)]
+					if l := len(uncBuf) + 8; l > len(msgBuf) {
+						msgBuf = make([]byte, l)
+					}
+
+					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
+					msgBuf = msgBuf[0 : len(uncBuf)+8]
+					copy(msgBuf[8:], uncBuf)
+
+					if debug {
+						l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
+					}
+				}
+			} else {
+				if debug {
+					l.Debugf("write empty message; %v", hm.hdr)
+				}
+				binary.BigEndian.PutUint32(msgBuf[4:8], 0)
+				msgBuf = msgBuf[:8]
+			}
+
+			binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))
+
+			if err == nil {
+				var n int
+				n, err = c.cw.Write(msgBuf)
+				if debug {
+					l.Debugf("wrote %d bytes on the wire", n)
+				}
+			}
+			if err != nil {
+				c.close(err)
+				return
+			}
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+func (c *rawConnection) close(err error) {
+	c.once.Do(func() {
+		close(c.closed)
+
+		c.awaitingMut.Lock()
+		for i, ch := range c.awaiting {
+			if ch != nil {
+				close(ch)
+				c.awaiting[i] = nil
+			}
+		}
+		c.awaitingMut.Unlock()
+
+		go c.receiver.Close(c.id, err)
+	})
+}
+
+func (c *rawConnection) idGenerator() {
+	nextID := 0
+	for {
+		nextID = (nextID + 1) & 0xfff
+		select {
+		case c.nextID <- nextID:
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+func (c *rawConnection) pingerLoop() {
+	var rc = make(chan bool, 1)
+	ticker := time.Tick(pingIdleTime / 2)
+	for {
+		select {
+		case <-ticker:
+			if d := time.Since(c.cr.Last()); d < pingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after rd", d)
+				}
+				continue
+			}
+			if d := time.Since(c.cw.Last()); d < pingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after wr", d)
+				}
+				continue
+			}
+			go func() {
+				if debug {
+					l.Debugln(c.id, "ping ->")
+				}
+				rc <- c.ping()
+			}()
+			select {
+			case ok := <-rc:
+				if debug {
+					l.Debugln(c.id, "<- pong")
+				}
+				if !ok {
+					c.close(fmt.Errorf("ping failure"))
+				}
+			case <-time.After(pingTimeout):
+				c.close(fmt.Errorf("ping timeout"))
+			case <-c.closed:
+				return
+			}
+
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+type Statistics struct {
+	At            time.Time
+	InBytesTotal  uint64
+	OutBytesTotal uint64
+}
+
+func (c *rawConnection) Statistics() Statistics {
+	return Statistics{
+		At:            time.Now(),
+		InBytesTotal:  c.cr.Tot(),
+		OutBytesTotal: c.cw.Tot(),
+	}
+}
+
+func IsDeleted(bits uint32) bool {
+	return bits&FlagDeleted != 0
+}
+
+func IsInvalid(bits uint32) bool {
+	return bits&FlagInvalid != 0
+}
+
+func IsDirectory(bits uint32) bool {
+	return bits&FlagDirectory != 0
+}
+
+func HasPermissionBits(bits uint32) bool {
+	return bits&FlagNoPermBits == 0
+}

internal/protocol/protocol_test.go

@@ -0,0 +1,383 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	"github.com/calmh/xdr"
+)
+
+var (
+	c0ID = NewNodeID([]byte{1})
+	c1ID = NewNodeID([]byte{2})
+)
+
+func TestHeaderFunctions(t *testing.T) {
+	f := func(ver, id, typ int) bool {
+		ver = int(uint(ver) % 16)
+		id = int(uint(id) % 4096)
+		typ = int(uint(typ) % 256)
+		h0 := header{version: ver, msgID: id, msgType: typ}
+		h1 := decodeHeader(encodeHeader(h0))
+		return h0 == h1
+	}
+	if err := quick.Check(f, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestHeaderLayout(t *testing.T) {
+	var e, a uint32
+
+	// Version are the first four bits
+	e = 0xf0000000
+	a = encodeHeader(header{version: 0xf})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Message ID are the following 12 bits
+	e = 0x0fff0000
+	a = encodeHeader(header{msgID: 0xfff})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Type are the last 8 bits before reserved
+	e = 0x0000ff00
+	a = encodeHeader(header{msgType: 0xff})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+}
+
+func TestPing(t *testing.T) {
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection)
+	c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection)
+
+	if ok := c0.ping(); !ok {
+		t.Error("c0 ping failed")
+	}
+	if ok := c1.ping(); !ok {
+		t.Error("c1 ping failed")
+	}
+}
+
+func TestPingErr(t *testing.T) {
+	e := errors.New("something broke")
+
+	for i := 0; i < 16; i++ {
+		for j := 0; j < 16; j++ {
+			m0 := newTestModel()
+			m1 := newTestModel()
+
+			ar, aw := io.Pipe()
+			br, bw := io.Pipe()
+			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
+			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
+
+			c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
+			NewConnection(c1ID, br, eaw, m1, "name", true)
+
+			res := c0.ping()
+			if (i < 8 || j < 8) && res {
+				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
+			} else if (i >= 12 && j >= 12) && !res {
+				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
+			}
+		}
+	}
+}
+
+// func TestRequestResponseErr(t *testing.T) {
+// 	e := errors.New("something broke")
+
+// 	var pass bool
+// 	for i := 0; i < 48; i++ {
+// 		for j := 0; j < 38; j++ {
+// 			m0 := newTestModel()
+// 			m0.data = []byte("response data")
+// 			m1 := newTestModel()
+
+// 			ar, aw := io.Pipe()
+// 			br, bw := io.Pipe()
+// 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
+// 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
+
+// 			NewConnection(c0ID, ar, ebw, m0, nil)
+// 			c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection)
+
+// 			d, err := c1.Request("default", "tn", 1234, 5678)
+// 			if err == e || err == ErrClosed {
+// 				t.Logf("Error at %d+%d bytes", i, j)
+// 				if !m1.isClosed() {
+// 					t.Fatal("c1 not closed")
+// 				}
+// 				if !m0.isClosed() {
+// 					t.Fatal("c0 not closed")
+// 				}
+// 				continue
+// 			}
+// 			if err != nil {
+// 				t.Fatal(err)
+// 			}
+// 			if string(d) != "response data" {
+// 				t.Fatalf("Incorrect response data %q", string(d))
+// 			}
+// 			if m0.repo != "default" {
+// 				t.Fatalf("Incorrect repo %q", m0.repo)
+// 			}
+// 			if m0.name != "tn" {
+// 				t.Fatalf("Incorrect name %q", m0.name)
+// 			}
+// 			if m0.offset != 1234 {
+// 				t.Fatalf("Incorrect offset %d", m0.offset)
+// 			}
+// 			if m0.size != 5678 {
+// 				t.Fatalf("Incorrect size %d", m0.size)
+// 			}
+// 			t.Logf("Pass at %d+%d bytes", i, j)
+// 			pass = true
+// 		}
+// 	}
+// 	if !pass {
+// 		t.Fatal("Never passed")
+// 	}
+// }
+
+func TestVersionErr(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", true)
+
+	w := xdr.NewWriter(c0.cw)
+	w.WriteUint32(encodeHeader(header{
+		version: 2,
+		msgID:   0,
+		msgType: 0,
+	}))
+	w.WriteUint32(0)
+
+	if !m1.isClosed() {
+		t.Error("Connection should close due to unknown version")
+	}
+}
+
+func TestTypeErr(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", true)
+
+	w := xdr.NewWriter(c0.cw)
+	w.WriteUint32(encodeHeader(header{
+		version: 0,
+		msgID:   0,
+		msgType: 42,
+	}))
+	w.WriteUint32(0)
+
+	if !m1.isClosed() {
+		t.Error("Connection should close due to unknown message type")
+	}
+}
+
+func TestClose(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", true)
+
+	c0.close(nil)
+
+	<-c0.closed
+	if !m0.isClosed() {
+		t.Fatal("Connection should be closed")
+	}
+
+	// None of these should panic, some should return an error
+
+	if c0.ping() {
+		t.Error("Ping should not return true")
+	}
+
+	c0.Index("default", nil)
+	c0.Index("default", nil)
+
+	if _, err := c0.Request("default", "foo", 0, 0); err == nil {
+		t.Error("Request should return an error")
+	}
+}
+
+func TestElementSizeExceededNested(t *testing.T) {
+	m := ClusterConfigMessage{
+		Repositories: []Repository{
+			{ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"},
+		},
+	}
+	_, err := m.EncodeXDR(ioutil.Discard)
+	if err == nil {
+		t.Errorf("ID length %d > max 64, but no error", len(m.Repositories[0].ID))
+	}
+}
+
+func TestMarshalIndexMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 IndexMessage) bool {
+		for _, f := range m1.Files {
+			for i := range f.Blocks {
+				f.Blocks[i].Offset = 0
+				if len(f.Blocks[i].Hash) == 0 {
+					f.Blocks[i].Hash = nil
+				}
+			}
+		}
+
+		return testMarshal(t, "index", &m1, &IndexMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalRequestMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 RequestMessage) bool {
+		return testMarshal(t, "request", &m1, &RequestMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalResponseMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 ResponseMessage) bool {
+		if len(m1.Data) == 0 {
+			m1.Data = nil
+		}
+		return testMarshal(t, "response", &m1, &ResponseMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalClusterConfigMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 ClusterConfigMessage) bool {
+		return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalCloseMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 CloseMessage) bool {
+		return testMarshal(t, "close", &m1, &CloseMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+type message interface {
+	EncodeXDR(io.Writer) (int, error)
+	DecodeXDR(io.Reader) error
+}
+
+func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
+	var buf bytes.Buffer
+
+	failed := func(bc []byte) {
+		bs, _ := json.MarshalIndent(m1, "", "  ")
+		ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
+		bs, _ = json.MarshalIndent(m2, "", "  ")
+		ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
+		if len(bc) > 0 {
+			f, _ := os.Create(prefix + "-data.txt")
+			fmt.Fprint(f, hex.Dump(bc))
+			f.Close()
+		}
+	}
+
+	_, err := m1.EncodeXDR(&buf)
+	if err == xdr.ErrElementSizeExceeded {
+		return true
+	}
+	if err != nil {
+		failed(nil)
+		t.Fatal(err)
+	}
+
+	bc := make([]byte, len(buf.Bytes()))
+	copy(bc, buf.Bytes())
+
+	err = m2.DecodeXDR(&buf)
+	if err != nil {
+		failed(bc)
+		t.Fatal(err)
+	}
+
+	ok := reflect.DeepEqual(m1, m2)
+	if !ok {
+		failed(bc)
+	}
+	return ok
+}

internal/protocol/wireformat.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+import (
+	"path/filepath"
+
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+type wireFormatConnection struct {
+	next Connection
+}
+
+func (c wireFormatConnection) ID() NodeID {
+	return c.next.ID()
+}
+
+func (c wireFormatConnection) Name() string {
+	return c.next.Name()
+}
+
+func (c wireFormatConnection) Index(repo string, fs []FileInfo) error {
+	var myFs = make([]FileInfo, len(fs))
+	copy(myFs, fs)
+
+	for i := range fs {
+		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
+	}
+
+	return c.next.Index(repo, myFs)
+}
+
+func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error {
+	var myFs = make([]FileInfo, len(fs))
+	copy(myFs, fs)
+
+	for i := range fs {
+		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
+	}
+
+	return c.next.IndexUpdate(repo, myFs)
+}
+
+func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) {
+	name = norm.NFC.String(filepath.ToSlash(name))
+	return c.next.Request(repo, name, offset, size)
+}
+
+func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) {
+	c.next.ClusterConfig(config)
+}
+
+func (c wireFormatConnection) Statistics() Statistics {
+	return c.next.Statistics()
+}

internal/scanner/blockqueue.go

@@ -0,0 +1,73 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+// The parallell hasher reads FileInfo structures from the inbox, hashes the
+// file to populate the Blocks element and sends it to the outbox. A number of
+// workers are used in parallel. The outbox will become closed when the inbox
+// is closed and all items handled.
+
+func newParallelHasher(dir string, blockSize, workers int, outbox, inbox chan protocol.FileInfo) {
+	var wg sync.WaitGroup
+	wg.Add(workers)
+
+	for i := 0; i < workers; i++ {
+		go func() {
+			hashFile(dir, blockSize, outbox, inbox)
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(outbox)
+	}()
+}
+
+func hashFile(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
+	for f := range inbox {
+		if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) {
+			outbox <- f
+			continue
+		}
+
+		fd, err := os.Open(filepath.Join(dir, f.Name))
+		if err != nil {
+			if debug {
+				l.Debugln("open:", err)
+			}
+			continue
+		}
+
+		fi, err := fd.Stat()
+		if err != nil {
+			fd.Close()
+			if debug {
+				l.Debugln("stat:", err)
+			}
+			continue
+		}
+		blocks, err := Blocks(fd, blockSize, fi.Size())
+		fd.Close()
+
+		if err != nil {
+			if debug {
+				l.Debugln("hash error:", f.Name, err)
+			}
+			continue
+		}
+
+		f.Blocks = blocks
+		outbox <- f
+	}
+}

internal/scanner/blocks.go

@@ -0,0 +1,90 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"io"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+const StandardBlockSize = 128 * 1024
+
+var sha256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
+
+// Blocks returns the blockwise hash of the reader.
+func Blocks(r io.Reader, blocksize int, sizehint int64) ([]protocol.BlockInfo, error) {
+	var blocks []protocol.BlockInfo
+	if sizehint > 0 {
+		blocks = make([]protocol.BlockInfo, 0, int(sizehint/int64(blocksize)))
+	}
+	var offset int64
+	hf := sha256.New()
+	for {
+		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
+		n, err := io.Copy(hf, lr)
+		if err != nil {
+			return nil, err
+		}
+
+		if n == 0 {
+			break
+		}
+
+		b := protocol.BlockInfo{
+			Size:   uint32(n),
+			Offset: offset,
+			Hash:   hf.Sum(nil),
+		}
+		blocks = append(blocks, b)
+		offset += int64(n)
+
+		hf.Reset()
+	}
+
+	if len(blocks) == 0 {
+		// Empty file
+		blocks = append(blocks, protocol.BlockInfo{
+			Offset: 0,
+			Size:   0,
+			Hash:   sha256OfNothing,
+		})
+	}
+
+	return blocks, nil
+}
+
+// BlockDiff returns lists of common and missing (to transform src into tgt)
+// blocks. Both block lists must have been created with the same block size.
+func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
+	if len(tgt) == 0 && len(src) != 0 {
+		return nil, nil
+	}
+
+	// Set the Offset field on each target block
+	var offset int64
+	for i := range tgt {
+		tgt[i].Offset = offset
+		offset += int64(tgt[i].Size)
+	}
+
+	if len(tgt) != 0 && len(src) == 0 {
+		// Copy the entire file
+		return nil, tgt
+	}
+
+	for i := range tgt {
+		if i >= len(src) || bytes.Compare(tgt[i].Hash, src[i].Hash) != 0 {
+			// Copy differing block
+			need = append(need, tgt[i])
+		} else {
+			have = append(have, tgt[i])
+		}
+	}
+
+	return have, need
+}

internal/scanner/blocks_test.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+var blocksTestData = []struct {
+	data      []byte
+	blocksize int
+	hash      []string
+}{
+	{[]byte(""), 1024, []string{
+		"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}},
+	{[]byte("contents"), 1024, []string{
+		"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
+	{[]byte("contents"), 9, []string{
+		"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
+	{[]byte("contents"), 8, []string{
+		"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
+	{[]byte("contents"), 7, []string{
+		"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
+		"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89"},
+	},
+	{[]byte("contents"), 3, []string{
+		"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
+		"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
+		"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
+	},
+	{[]byte("conconts"), 3, []string{
+		"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
+		"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
+		"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
+	},
+	{[]byte("contenten"), 3, []string{
+		"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
+		"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
+		"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3"},
+	},
+}
+
+func TestBlocks(t *testing.T) {
+	for _, test := range blocksTestData {
+		buf := bytes.NewBuffer(test.data)
+		blocks, err := Blocks(buf, test.blocksize, 0)
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if l := len(blocks); l != len(test.hash) {
+			t.Fatalf("Incorrect number of blocks %d != %d", l, len(test.hash))
+		} else {
+			i := 0
+			for off := int64(0); off < int64(len(test.data)); off += int64(test.blocksize) {
+				if blocks[i].Offset != off {
+					t.Errorf("Incorrect offset for block %d: %d != %d", i, blocks[i].Offset, off)
+				}
+
+				bs := test.blocksize
+				if rem := len(test.data) - int(off); bs > rem {
+					bs = rem
+				}
+				if int(blocks[i].Size) != bs {
+					t.Errorf("Incorrect length for block %d: %d != %d", i, blocks[i].Size, bs)
+				}
+				if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
+					t.Errorf("Incorrect block hash %q != %q", h, test.hash[i])
+				}
+
+				i++
+			}
+		}
+	}
+}
+
+var diffTestData = []struct {
+	a string
+	b string
+	s int
+	d []protocol.BlockInfo
+}{
+	{"contents", "contents", 1024, []protocol.BlockInfo{}},
+	{"", "", 1024, []protocol.BlockInfo{}},
+	{"contents", "contents", 3, []protocol.BlockInfo{}},
+	{"contents", "cantents", 3, []protocol.BlockInfo{{0, 3, nil}}},
+	{"contents", "contants", 3, []protocol.BlockInfo{{3, 3, nil}}},
+	{"contents", "cantants", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}}},
+	{"contents", "", 3, []protocol.BlockInfo{{0, 0, nil}}},
+	{"", "contents", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
+	{"con", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
+	{"contents", "con", 3, nil},
+	{"contents", "cont", 3, []protocol.BlockInfo{{3, 1, nil}}},
+	{"cont", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
+}
+
+func TestDiff(t *testing.T) {
+	for i, test := range diffTestData {
+		a, _ := Blocks(bytes.NewBufferString(test.a), test.s, 0)
+		b, _ := Blocks(bytes.NewBufferString(test.b), test.s, 0)
+		_, d := BlockDiff(a, b)
+		if len(d) != len(test.d) {
+			t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
+		} else {
+			for j := range test.d {
+				if d[j].Offset != test.d[j].Offset {
+					t.Errorf("Incorrect offset for diff %d block %d; %d != %d", i, j, d[j].Offset, test.d[j].Offset)
+				}
+				if d[j].Size != test.d[j].Size {
+					t.Errorf("Incorrect length for diff %d block %d; %d != %d", i, j, d[j].Size, test.d[j].Size)
+				}
+			}
+		}
+	}
+}

internal/scanner/debug.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+	"os"
+	"strings"
+
+	"github.com/syncthing/syncthing/internal/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "scanner") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

internal/scanner/doc.go

@@ -0,0 +1,6 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// Package scanner implements a file system scanner and hasher.
+package scanner

internal/scanner/testdata/.stignore

@@ -0,0 +1,4 @@
+#include excludes
+
+bfile
+dir1/cfile

internal/scanner/testdata/afile

@@ -0,0 +1 @@
+foo

internal/scanner/testdata/bfile

@@ -0,0 +1 @@
+bar

internal/scanner/testdata/dir1/cfile

@@ -0,0 +1 @@
+baz

internal/scanner/testdata/dir1/dfile

@@ -0,0 +1 @@
+quux

internal/scanner/testdata/dir2/cfile

@@ -0,0 +1 @@
+baz

internal/scanner/testdata/dir2/dfile

@@ -0,0 +1 @@
+quux