From 2cfb24892fcc420f1ad4197d66076cdd1511ba34 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 7 Jan 2014 22:44:21 +0100 Subject: [PATCH] Add version and invalid bit to protocol --- model/model.go | 52 +++++++++++++++++++++++---------------- model/model_puller.go | 3 ++- model/model_test.go | 26 ++++++++++++++------ model/walk.go | 19 ++++++++++++-- protocol/PROTOCOL.md | 28 +++++++++++++-------- protocol/messages.go | 2 ++ protocol/messages_test.go | 6 ++++- protocol/protocol.go | 6 +++++ 8 files changed, 100 insertions(+), 42 deletions(-) diff --git a/model/model.go b/model/model.go index 33ac9c42..7e434dd5 100644 --- a/model/model.go +++ b/model/model.go @@ -56,8 +56,6 @@ type Model struct { } const ( - FlagDeleted = 1 << 12 - idxBcastHoldtime = 15 * time.Second // Wait at least this long after the last index modification idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long @@ -65,7 +63,10 @@ const ( maxFileHoldTimeS = 600 // Always allow file changes at least this often ) -var ErrNoSuchFile = errors.New("no such file") +var ( + ErrNoSuchFile = errors.New("no such file") + ErrInvalid = errors.New("file is invalid") +) // NewModel creates and starts a new model. The model starts in read-only mode, // where it sends index information to connected peers and responds to requests @@ -159,7 +160,7 @@ func (m *Model) GlobalSize() (files, deleted, bytes int) { defer m.RUnlock() for _, f := range m.global { - if f.Flags&FlagDeleted == 0 { + if f.Flags&protocol.FlagDeleted == 0 { files++ bytes += f.Size() } else { @@ -176,7 +177,7 @@ func (m *Model) LocalSize() (files, deleted, bytes int) { defer m.RUnlock() for _, f := range m.local { - if f.Flags&FlagDeleted == 0 { + if f.Flags&protocol.FlagDeleted == 0 { files++ bytes += f.Size() } else { @@ -193,7 +194,7 @@ func (m *Model) InSyncSize() (files, bytes int) { defer m.RUnlock() for n, f := range m.local { - if gf, ok := m.global[n]; ok && f.Modified == gf.Modified { + if gf, ok := m.global[n]; ok && f.Equals(gf) { files++ bytes += f.Size() } @@ -249,7 +250,7 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) { } for _, f := range fs { - if f.Flags&FlagDeleted != 0 && !m.delete { + if f.Flags&protocol.FlagDeleted != 0 && !m.delete { // Files marked as deleted do not even enter the model continue } @@ -284,13 +285,16 @@ func (m *Model) Close(node string, err error) { func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) { // Verify that the requested file exists in the local and global model. m.RLock() - _, localOk := m.local[name] + lf, localOk := m.local[name] _, globalOk := m.global[name] m.RUnlock() if !localOk || !globalOk { log.Printf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash) return nil, ErrNoSuchFile } + if lf.Flags&protocol.FlagInvalid != 0 { + return nil, ErrInvalid + } if m.trace["net"] && nodeID != "" { log.Printf("NET REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash) @@ -322,7 +326,7 @@ func (m *Model) ReplaceLocal(fs []File) { for _, f := range fs { newLocal[f.Name] = f - if ef := m.local[f.Name]; ef.Modified != f.Modified { + if ef := m.local[f.Name]; !ef.Equals(f) { updated = true } } @@ -430,7 +434,7 @@ func (m *Model) protocolIndex() []protocol.FileInfo { mf := fileInfoFromFile(f) if m.trace["idx"] { var flagComment string - if mf.Flags&FlagDeleted != 0 { + if mf.Flags&protocol.FlagDeleted != 0 { flagComment = " (deleted)" } log.Printf("IDX: %q m=%d f=%o%s (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, len(mf.Blocks)) @@ -496,10 +500,10 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool { var updated bool for n, f := range m.local { if _, ok := newLocal[n]; !ok { - if gf := m.global[n]; gf.Modified <= f.Modified { - if f.Flags&FlagDeleted == 0 { - f.Flags = FlagDeleted - f.Modified = f.Modified + 1 + if gf := m.global[n]; !gf.NewerThan(f) { + if f.Flags&protocol.FlagDeleted == 0 { + f.Flags = protocol.FlagDeleted + f.Version++ f.Blocks = nil updated = true } @@ -511,7 +515,7 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool { } func (m *Model) updateLocal(f File) { - if ef, ok := m.local[f.Name]; !ok || ef.Modified != f.Modified { + if ef, ok := m.local[f.Name]; !ok || !ef.Equals(f) { m.local[f.Name] = f m.recomputeGlobal() m.recomputeNeed() @@ -530,7 +534,7 @@ func (m *Model) recomputeGlobal() { for _, fs := range m.remote { for n, f := range fs { - if cf, ok := newGlobal[n]; !ok || cf.Modified < f.Modified { + if cf, ok := newGlobal[n]; !ok || f.NewerThan(cf) { newGlobal[n] = f } } @@ -543,7 +547,7 @@ func (m *Model) recomputeGlobal() { updated = true } else { for n, f0 := range newGlobal { - if f1, ok := m.global[n]; !ok || f0.Modified != f1.Modified { + if f1, ok := m.global[n]; !ok || !f0.Equals(f1) { updated = true break } @@ -561,12 +565,16 @@ func (m *Model) recomputeNeed() { m.need = make(map[string]bool) for n, f := range m.global { hf, ok := m.local[n] - if !ok || f.Modified > hf.Modified { - if f.Flags&FlagDeleted != 0 && !m.delete { + if !ok || f.NewerThan(hf) { + if f.Flags&protocol.FlagInvalid != 0 { + // Never attempt to sync invalid files + continue + } + if f.Flags&protocol.FlagDeleted != 0 && !m.delete { // Don't want to delete files, so forget this need continue } - if f.Flags&FlagDeleted != 0 && !ok { + if f.Flags&protocol.FlagDeleted != 0 && !ok { // Don't have the file, so don't need to delete it continue } @@ -584,7 +592,7 @@ func (m *Model) whoHas(name string) []string { gf := m.global[name] for node, files := range m.remote { - if file, ok := files[name]; ok && file.Modified == gf.Modified { + if file, ok := files[name]; ok && file.Equals(gf) { remote = append(remote, node) } } @@ -607,6 +615,7 @@ func fileFromFileInfo(f protocol.FileInfo) File { Name: f.Name, Flags: f.Flags, Modified: int64(f.Modified), + Version: f.Version, Blocks: blocks, } } @@ -623,6 +632,7 @@ func fileInfoFromFile(f File) protocol.FileInfo { Name: f.Name, Flags: f.Flags, Modified: int64(f.Modified), + Version: f.Version, Blocks: blocks, } } diff --git a/model/model_puller.go b/model/model_puller.go index 0f628f48..42fc8bbe 100644 --- a/model/model_puller.go +++ b/model/model_puller.go @@ -25,6 +25,7 @@ import ( "time" "github.com/calmh/syncthing/buffers" + "github.com/calmh/syncthing/protocol" ) func (m *Model) pullFile(name string) error { @@ -171,7 +172,7 @@ func (m *Model) puller() { } var err error - if f.Flags&FlagDeleted == 0 { + if f.Flags&protocol.FlagDeleted == 0 { if m.trace["file"] { log.Printf("FILE: Pull %q", n) } diff --git a/model/model_test.go b/model/model_test.go index b3293969..4ee1dd09 100644 --- a/model/model_test.go +++ b/model/model_test.go @@ -228,9 +228,12 @@ func TestDelete(t *testing.T) { if len(m.local["a new file"].Blocks) != 0 { t.Error("Unexpected non-zero blocks for deleted file in local") } - if ft := m.local["a new file"].Modified; ft != ot+1 { + if ft := m.local["a new file"].Modified; ft != ot { t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1) } + if fv := m.local["a new file"].Version; fv != 1 { + t.Errorf("Unexpected version %d != 1 for deleted file in local", fv) + } if m.global["a new file"].Flags&(1<<12) == 0 { t.Error("Unexpected deleted flag = 0 in global table") @@ -238,8 +241,11 @@ func TestDelete(t *testing.T) { if len(m.global["a new file"].Blocks) != 0 { t.Error("Unexpected non-zero blocks for deleted file in global") } - if ft := m.local["a new file"].Modified; ft != ot+1 { - t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1) + if ft := m.global["a new file"].Modified; ft != ot { + t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot+1) + } + if fv := m.local["a new file"].Version; fv != 1 { + t.Errorf("Unexpected version %d != 1 for deleted file in global", fv) } // Another update should change nothing @@ -259,8 +265,11 @@ func TestDelete(t *testing.T) { if len(m.local["a new file"].Blocks) != 0 { t.Error("Unexpected non-zero blocks for deleted file in local") } - if ft := m.local["a new file"].Modified; ft != ot+1 { - t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1) + if ft := m.local["a new file"].Modified; ft != ot { + t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot) + } + if fv := m.local["a new file"].Version; fv != 1 { + t.Errorf("Unexpected version %d != 1 for deleted file in local", fv) } if m.global["a new file"].Flags&(1<<12) == 0 { @@ -269,8 +278,11 @@ func TestDelete(t *testing.T) { if len(m.global["a new file"].Blocks) != 0 { t.Error("Unexpected non-zero blocks for deleted file in global") } - if ft := m.local["a new file"].Modified; ft != ot+1 { - t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1) + if ft := m.global["a new file"].Modified; ft != ot { + t.Errorf("Unexpected time %d != %d for deleted file in global", ft, ot) + } + if fv := m.local["a new file"].Version; fv != 1 { + t.Errorf("Unexpected version %d != 1 for deleted file in global", fv) } } diff --git a/model/walk.go b/model/walk.go index 58da2315..52b7d10b 100644 --- a/model/walk.go +++ b/model/walk.go @@ -10,6 +10,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/calmh/syncthing/protocol" ) const BlockSize = 128 * 1024 @@ -18,6 +20,7 @@ type File struct { Name string Flags uint32 Modified int64 + Version uint32 Blocks []Block } @@ -28,6 +31,14 @@ func (f File) Size() (bytes int) { return } +func (f File) Equals(o File) bool { + return f.Modified == o.Modified && f.Version == o.Version +} + +func (f File) NewerThan(o File) bool { + return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version) +} + func isTempName(name string) bool { return strings.HasPrefix(path.Base(name), ".syncthing.") } @@ -79,7 +90,10 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun m.RUnlock() if ok && hf.Modified == modified { - // No change + if nf := uint32(info.Mode()); nf != hf.Flags { + hf.Flags = nf + hf.Version++ + } *res = append(*res, hf) } else { m.Lock() @@ -89,7 +103,8 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun } if ok { - // Files that are ignored will be suppressed but don't actually exist in the local model + hf.Flags = protocol.FlagInvalid + hf.Version++ *res = append(*res, hf) } m.Unlock() diff --git a/protocol/PROTOCOL.md b/protocol/PROTOCOL.md index 9a2c264f..8fb090ab 100644 --- a/protocol/PROTOCOL.md +++ b/protocol/PROTOCOL.md @@ -62,11 +62,10 @@ reserved bits must be set to zero. All data following the message header is in XDR (RFC 1014) encoding. The actual data types in use by BEP, in XDR naming convention, are: - - unsigned int -- unsigned 32 bit integer - - hyper -- signed 64 bit integer - - unsigned hyper -- signed 64 bit integer - - opaque<> -- variable length opaque data - - string<> -- variable length string + - (unsigned) int -- (unsigned) 32 bit integer + - (unsigned) hyper -- (unsigned) 64 bit integer + - opaque<> -- variable length opaque data + - string<> -- variable length string The encoding of opaque<> and string<> are identical, the distinction is solely in interpretation. Opaque data should not be interpreted as such, @@ -92,6 +91,7 @@ message. string Name<>; unsigned int Flags; hyper Modified; + unsigned int Version; BlockInfo Blocks<>; } @@ -102,15 +102,19 @@ message. The file name is the part relative to the repository root. The modification time is expressed as the number of seconds since the Unix -Epoch. The hash algorithm is implied by the hash length. Currently, the -hash must be 32 bytes long and computed by SHA256. +Epoch. The version field is a counter that increments each time the file +changes but resets to zero each time the modification is updated. This +is used to signal changes to the file (or file metadata) while the +modification time remains unchanged. The hash algorithm is implied by +the hash length. Currently, the hash must be 32 bytes long and computed +by SHA256. The flags field is made up of the following single bit flags: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Reserved |D| Unix Perm. & Mode | + | Reserved |I|D| Unix Perm. & Mode | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - The lower 12 bits hold the common Unix permission and mode bits. @@ -118,9 +122,13 @@ The flags field is made up of the following single bit flags: - Bit 19 ("D") is set when the file has been deleted. The block list shall contain zero blocks and the modification time indicates the time of deletion or, if deletion time is not reliably determinable, - one second past the last know modification time. + the last known modification time and a higher version number. - - Bit 0 through 18 are reserved for future use and shall be set to + - Bit 18 ("I") is set when the file is invalid and unavailable for + synchronization. A peer may set this bit to indicate that it can + temporarily not serve data for the file. + + - Bit 0 through 17 are reserved for future use and shall be set to zero. ### Request (Type = 2) diff --git a/protocol/messages.go b/protocol/messages.go index 9b273a22..66371130 100644 --- a/protocol/messages.go +++ b/protocol/messages.go @@ -39,6 +39,7 @@ func (w *marshalWriter) writeIndex(idx []FileInfo) { w.writeString(f.Name) w.writeUint32(f.Flags) w.writeUint64(uint64(f.Modified)) + w.writeUint32(f.Version) w.writeUint32(uint32(len(f.Blocks))) for _, b := range f.Blocks { w.writeUint32(b.Length) @@ -77,6 +78,7 @@ func (r *marshalReader) readIndex() []FileInfo { files[i].Name = r.readString() files[i].Flags = r.readUint32() files[i].Modified = int64(r.readUint64()) + files[i].Version = r.readUint32() nblocks := r.readUint32() blocks := make([]BlockInfo, nblocks) for j := range blocks { diff --git a/protocol/messages_test.go b/protocol/messages_test.go index b4d74fc4..46e4feca 100644 --- a/protocol/messages_test.go +++ b/protocol/messages_test.go @@ -12,8 +12,9 @@ func TestIndex(t *testing.T) { idx := []FileInfo{ { "Foo", - 0755, + FlagInvalid & FlagDeleted & 0755, 1234567890, + 142, []BlockInfo{ {12345678, []byte("hash hash hash")}, {23456781, []byte("ash hash hashh")}, @@ -23,6 +24,7 @@ func TestIndex(t *testing.T) { "Quux/Quux", 0644, 2345678901, + 232323232, []BlockInfo{ {45678123, []byte("4321 hash hash hash")}, {56781234, []byte("3214 ash hash hashh")}, @@ -81,6 +83,7 @@ func BenchmarkWriteIndex(b *testing.B) { "Foo", 0777, 1234567890, + 424242, []BlockInfo{ {12345678, []byte("hash hash hash")}, {23456781, []byte("ash hash hashh")}, @@ -90,6 +93,7 @@ func BenchmarkWriteIndex(b *testing.B) { "Quux/Quux", 0644, 2345678901, + 323232, []BlockInfo{ {45678123, []byte("4321 hash hash hash")}, {56781234, []byte("3214 ash hash hashh")}, diff --git a/protocol/protocol.go b/protocol/protocol.go index 0dc4e3ce..cd87fd4f 100644 --- a/protocol/protocol.go +++ b/protocol/protocol.go @@ -20,10 +20,16 @@ const ( messageTypeIndexUpdate = 6 ) +const ( + FlagDeleted = 1 << 12 + FlagInvalid = 1 << 13 +) + type FileInfo struct { Name string Flags uint32 Modified int64 + Version uint32 Blocks []BlockInfo }