Add version and invalid bit to protocol

model/model.go

@@ -56,8 +56,6 @@ type Model struct {
 }
 
 const (
-	FlagDeleted = 1 << 12
-
 	idxBcastHoldtime = 15 * time.Second  // Wait at least this long after the last index modification
 	idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
 
@@ -65,7 +63,10 @@ const (
 	maxFileHoldTimeS = 600 // Always allow file changes at least this often
 )
 
-var ErrNoSuchFile = errors.New("no such file")
+var (
+	ErrNoSuchFile = errors.New("no such file")
+	ErrInvalid    = errors.New("file is invalid")
+)
 
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
@@ -159,7 +160,7 @@ func (m *Model) GlobalSize() (files, deleted, bytes int) {
 	defer m.RUnlock()
 
 	for _, f := range m.global {
-		if f.Flags&FlagDeleted == 0 {
+		if f.Flags&protocol.FlagDeleted == 0 {
 			files++
 			bytes += f.Size()
 		} else {
@@ -176,7 +177,7 @@ func (m *Model) LocalSize() (files, deleted, bytes int) {
 	defer m.RUnlock()
 
 	for _, f := range m.local {
-		if f.Flags&FlagDeleted == 0 {
+		if f.Flags&protocol.FlagDeleted == 0 {
 			files++
 			bytes += f.Size()
 		} else {
@@ -193,7 +194,7 @@ func (m *Model) InSyncSize() (files, bytes int) {
 	defer m.RUnlock()
 
 	for n, f := range m.local {
-		if gf, ok := m.global[n]; ok && f.Modified == gf.Modified {
+		if gf, ok := m.global[n]; ok && f.Equals(gf) {
 			files++
 			bytes += f.Size()
 		}
@@ -249,7 +250,7 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
 	}
 
 	for _, f := range fs {
-		if f.Flags&FlagDeleted != 0 && !m.delete {
+		if f.Flags&protocol.FlagDeleted != 0 && !m.delete {
 			// Files marked as deleted do not even enter the model
 			continue
 		}
@@ -284,13 +285,16 @@ func (m *Model) Close(node string, err error) {
 func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
 	// Verify that the requested file exists in the local and global model.
 	m.RLock()
-	_, localOk := m.local[name]
+	lf, localOk := m.local[name]
 	_, globalOk := m.global[name]
 	m.RUnlock()
 	if !localOk || !globalOk {
 		log.Printf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
 		return nil, ErrNoSuchFile
 	}
+	if lf.Flags&protocol.FlagInvalid != 0 {
+		return nil, ErrInvalid
+	}
 
 	if m.trace["net"] && nodeID != "<local>" {
 		log.Printf("NET REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
@@ -322,7 +326,7 @@ func (m *Model) ReplaceLocal(fs []File) {
 
 	for _, f := range fs {
 		newLocal[f.Name] = f
-		if ef := m.local[f.Name]; ef.Modified != f.Modified {
+		if ef := m.local[f.Name]; !ef.Equals(f) {
 			updated = true
 		}
 	}
@@ -430,7 +434,7 @@ func (m *Model) protocolIndex() []protocol.FileInfo {
 		mf := fileInfoFromFile(f)
 		if m.trace["idx"] {
 			var flagComment string
-			if mf.Flags&FlagDeleted != 0 {
+			if mf.Flags&protocol.FlagDeleted != 0 {
 				flagComment = " (deleted)"
 			}
 			log.Printf("IDX: %q m=%d f=%o%s (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, len(mf.Blocks))
@@ -496,10 +500,10 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
 	var updated bool
 	for n, f := range m.local {
 		if _, ok := newLocal[n]; !ok {
-			if gf := m.global[n]; gf.Modified <= f.Modified {
+			if gf := m.global[n]; !gf.NewerThan(f) {
-				if f.Flags&FlagDeleted == 0 {
+				if f.Flags&protocol.FlagDeleted == 0 {
-					f.Flags = FlagDeleted
+					f.Flags = protocol.FlagDeleted
-					f.Modified = f.Modified + 1
+					f.Version++
 					f.Blocks = nil
 					updated = true
 				}
@@ -511,7 +515,7 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
 }
 
 func (m *Model) updateLocal(f File) {
-	if ef, ok := m.local[f.Name]; !ok || ef.Modified != f.Modified {
+	if ef, ok := m.local[f.Name]; !ok || !ef.Equals(f) {
 		m.local[f.Name] = f
 		m.recomputeGlobal()
 		m.recomputeNeed()
@@ -530,7 +534,7 @@ func (m *Model) recomputeGlobal() {
 
 	for _, fs := range m.remote {
 		for n, f := range fs {
-			if cf, ok := newGlobal[n]; !ok || cf.Modified < f.Modified {
+			if cf, ok := newGlobal[n]; !ok || f.NewerThan(cf) {
 				newGlobal[n] = f
 			}
 		}
@@ -543,7 +547,7 @@ func (m *Model) recomputeGlobal() {
 		updated = true
 	} else {
 		for n, f0 := range newGlobal {
-			if f1, ok := m.global[n]; !ok || f0.Modified != f1.Modified {
+			if f1, ok := m.global[n]; !ok || !f0.Equals(f1) {
 				updated = true
 				break
 			}
@@ -561,12 +565,16 @@ func (m *Model) recomputeNeed() {
 	m.need = make(map[string]bool)
 	for n, f := range m.global {
 		hf, ok := m.local[n]
-		if !ok || f.Modified > hf.Modified {
+		if !ok || f.NewerThan(hf) {
-			if f.Flags&FlagDeleted != 0 && !m.delete {
+			if f.Flags&protocol.FlagInvalid != 0 {
+				// Never attempt to sync invalid files
+				continue
+			}
+			if f.Flags&protocol.FlagDeleted != 0 && !m.delete {
 				// Don't want to delete files, so forget this need
 				continue
 			}
-			if f.Flags&FlagDeleted != 0 && !ok {
+			if f.Flags&protocol.FlagDeleted != 0 && !ok {
 				// Don't have the file, so don't need to delete it
 				continue
 			}
@@ -584,7 +592,7 @@ func (m *Model) whoHas(name string) []string {
 
 	gf := m.global[name]
 	for node, files := range m.remote {
-		if file, ok := files[name]; ok && file.Modified == gf.Modified {
+		if file, ok := files[name]; ok && file.Equals(gf) {
 			remote = append(remote, node)
 		}
 	}
@@ -607,6 +615,7 @@ func fileFromFileInfo(f protocol.FileInfo) File {
 		Name:     f.Name,
 		Flags:    f.Flags,
 		Modified: int64(f.Modified),
+		Version:  f.Version,
 		Blocks:   blocks,
 	}
 }
@@ -623,6 +632,7 @@ func fileInfoFromFile(f File) protocol.FileInfo {
 		Name:     f.Name,
 		Flags:    f.Flags,
 		Modified: int64(f.Modified),
+		Version:  f.Version,
 		Blocks:   blocks,
 	}
 }

model/model_puller.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/calmh/syncthing/buffers"
+	"github.com/calmh/syncthing/protocol"
 )
 
 func (m *Model) pullFile(name string) error {
@@ -171,7 +172,7 @@ func (m *Model) puller() {
 				}
 
 				var err error
-				if f.Flags&FlagDeleted == 0 {
+				if f.Flags&protocol.FlagDeleted == 0 {
 					if m.trace["file"] {
 						log.Printf("FILE: Pull %q", n)
 					}

model/model_test.go

@@ -228,9 +228,12 @@ func TestDelete(t *testing.T) {
 	if len(m.local["a new file"].Blocks) != 0 {
 		t.Error("Unexpected non-zero blocks for deleted file in local")
 	}
-	if ft := m.local["a new file"].Modified; ft != ot+1 {
+	if ft := m.local["a new file"].Modified; ft != ot {
 		t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
 	}
+	if fv := m.local["a new file"].Version; fv != 1 {
+		t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
+	}
 
 	if m.global["a new file"].Flags&(1<<12) == 0 {
 		t.Error("Unexpected deleted flag = 0 in global table")
@@ -238,8 +241,11 @@ func TestDelete(t *testing.T) {
 	if len(m.global["a new file"].Blocks) != 0 {
 		t.Error("Unexpected non-zero blocks for deleted file in global")
 	}
-	if ft := m.local["a new file"].Modified; ft != ot+1 {
+	if ft := m.global["a new file"].Modified; ft != ot {
-		t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
+		t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot+1)
+	}
+	if fv := m.local["a new file"].Version; fv != 1 {
+		t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
 	}
 
 	// Another update should change nothing
@@ -259,8 +265,11 @@ func TestDelete(t *testing.T) {
 	if len(m.local["a new file"].Blocks) != 0 {
 		t.Error("Unexpected non-zero blocks for deleted file in local")
 	}
-	if ft := m.local["a new file"].Modified; ft != ot+1 {
+	if ft := m.local["a new file"].Modified; ft != ot {
-		t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
+		t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot)
+	}
+	if fv := m.local["a new file"].Version; fv != 1 {
+		t.Errorf("Unexpected version %d != 1 for deleted file in local", fv)
 	}
 
 	if m.global["a new file"].Flags&(1<<12) == 0 {
@@ -269,8 +278,11 @@ func TestDelete(t *testing.T) {
 	if len(m.global["a new file"].Blocks) != 0 {
 		t.Error("Unexpected non-zero blocks for deleted file in global")
 	}
-	if ft := m.local["a new file"].Modified; ft != ot+1 {
+	if ft := m.global["a new file"].Modified; ft != ot {
-		t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
+		t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot)
+	}
+	if fv := m.local["a new file"].Version; fv != 1 {
+		t.Errorf("Unexpected version %d != 1 for deleted file in global", fv)
 	}
 }
 

model/walk.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/calmh/syncthing/protocol"
 )
 
 const BlockSize = 128 * 1024
@@ -18,6 +20,7 @@ type File struct {
 	Name     string
 	Flags    uint32
 	Modified int64
+	Version  uint32
 	Blocks   []Block
 }
 
@@ -28,6 +31,14 @@ func (f File) Size() (bytes int) {
 	return
 }
 
+func (f File) Equals(o File) bool {
+	return f.Modified == o.Modified && f.Version == o.Version
+}
+
+func (f File) NewerThan(o File) bool {
+	return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
+}
+
 func isTempName(name string) bool {
 	return strings.HasPrefix(path.Base(name), ".syncthing.")
 }
@@ -79,7 +90,10 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun
 			m.RUnlock()
 
 			if ok && hf.Modified == modified {
-				// No change
+				if nf := uint32(info.Mode()); nf != hf.Flags {
+					hf.Flags = nf
+					hf.Version++
+				}
 				*res = append(*res, hf)
 			} else {
 				m.Lock()
@@ -89,7 +103,8 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun
 					}
 
 					if ok {
-						// Files that are ignored will be suppressed but don't actually exist in the local model
+						hf.Flags = protocol.FlagInvalid
+						hf.Version++
 						*res = append(*res, hf)
 					}
 					m.Unlock()

protocol/PROTOCOL.md

@@ -62,9 +62,8 @@ reserved bits must be set to zero.
 All data following the message header is in XDR (RFC 1014) encoding.
 The actual data types in use by BEP, in XDR naming convention, are:
 
- - unsigned int   -- unsigned 32 bit integer
+ - (unsigned) int   -- (unsigned) 32 bit integer
- - hyper          -- signed 64 bit integer
+ - (unsigned) hyper -- (unsigned) 64 bit integer
- - unsigned hyper -- signed 64 bit integer
  - opaque<>         -- variable length opaque data
  - string<>         -- variable length string
 
@@ -92,6 +91,7 @@ message.
         string Name<>;
         unsigned int Flags;
         hyper Modified;
+        unsigned int Version;
         BlockInfo Blocks<>;
     }
 
@@ -102,15 +102,19 @@ message.
 
 The file name is the part relative to the repository root. The
 modification time is expressed as the number of seconds since the Unix
-Epoch. The hash algorithm is implied by the hash length. Currently, the
+Epoch. The version field is a counter that increments each time the file
-hash must be 32 bytes long and computed by SHA256.
+changes but resets to zero each time the modification is updated. This
+is used to signal changes to the file (or file metadata) while the
+modification time remains unchanged. The hash algorithm is implied by
+the hash length. Currently, the hash must be 32 bytes long and computed
+by SHA256.
 
 The flags field is made up of the following single bit flags:
 
      0                   1                   2                   3
      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |              Reserved               |D|   Unix Perm. & Mode   |
+    |              Reserved             |I|D|   Unix Perm. & Mode   |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
  - The lower 12 bits hold the common Unix permission and mode bits.
@@ -118,9 +122,13 @@ The flags field is made up of the following single bit flags:
  - Bit 19 ("D") is set when the file has been deleted. The block list
    shall contain zero blocks and the modification time indicates the
    time of deletion or, if deletion time is not reliably determinable,
-   one second past the last know modification time.
+   the last known modification time and a higher version number.
 
- - Bit 0 through 18 are reserved for future use and shall be set to
+ - Bit 18 ("I") is set when the file is invalid and unavailable for
+   synchronization. A peer may set this bit to indicate that it can
+   temporarily not serve data for the file.
+
+ - Bit 0 through 17 are reserved for future use and shall be set to
    zero.
 
 ### Request (Type = 2)

protocol/messages.go

@@ -39,6 +39,7 @@ func (w *marshalWriter) writeIndex(idx []FileInfo) {
 		w.writeString(f.Name)
 		w.writeUint32(f.Flags)
 		w.writeUint64(uint64(f.Modified))
+		w.writeUint32(f.Version)
 		w.writeUint32(uint32(len(f.Blocks)))
 		for _, b := range f.Blocks {
 			w.writeUint32(b.Length)
@@ -77,6 +78,7 @@ func (r *marshalReader) readIndex() []FileInfo {
 			files[i].Name = r.readString()
 			files[i].Flags = r.readUint32()
 			files[i].Modified = int64(r.readUint64())
+			files[i].Version = r.readUint32()
 			nblocks := r.readUint32()
 			blocks := make([]BlockInfo, nblocks)
 			for j := range blocks {

protocol/messages_test.go

@@ -12,8 +12,9 @@ func TestIndex(t *testing.T) {
 	idx := []FileInfo{
 		{
 			"Foo",
-			0755,
+			FlagInvalid & FlagDeleted & 0755,
 			1234567890,
+			142,
 			[]BlockInfo{
 				{12345678, []byte("hash hash hash")},
 				{23456781, []byte("ash hash hashh")},
@@ -23,6 +24,7 @@ func TestIndex(t *testing.T) {
 			"Quux/Quux",
 			0644,
 			2345678901,
+			232323232,
 			[]BlockInfo{
 				{45678123, []byte("4321 hash hash hash")},
 				{56781234, []byte("3214 ash hash hashh")},
@@ -81,6 +83,7 @@ func BenchmarkWriteIndex(b *testing.B) {
 			"Foo",
 			0777,
 			1234567890,
+			424242,
 			[]BlockInfo{
 				{12345678, []byte("hash hash hash")},
 				{23456781, []byte("ash hash hashh")},
@@ -90,6 +93,7 @@ func BenchmarkWriteIndex(b *testing.B) {
 			"Quux/Quux",
 			0644,
 			2345678901,
+			323232,
 			[]BlockInfo{
 				{45678123, []byte("4321 hash hash hash")},
 				{56781234, []byte("3214 ash hash hashh")},

protocol/protocol.go

@@ -20,10 +20,16 @@ const (
 	messageTypeIndexUpdate = 6
 )
 
+const (
+	FlagDeleted = 1 << 12
+	FlagInvalid = 1 << 13
+)
+
 type FileInfo struct {
 	Name     string
 	Flags    uint32
 	Modified int64
+	Version  uint32
 	Blocks   []BlockInfo
 }