From 941f637bca8b64c46cfa79edc0788375a2a26f01 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 26 Mar 2015 21:49:49 +0000 Subject: [PATCH 1/5] Update protocol package --- Godeps/Godeps.json | 2 +- .../syncthing/protocol/common_test.go | 12 ++- .../github.com/syncthing/protocol/errors.go | 51 +++++++++++++ .../github.com/syncthing/protocol/message.go | 5 +- .../syncthing/protocol/message_xdr.go | 8 +- .../syncthing/protocol/nativemodel_darwin.go | 12 +-- .../syncthing/protocol/nativemodel_unix.go | 12 +-- .../syncthing/protocol/nativemodel_windows.go | 51 ++++++------- .../github.com/syncthing/protocol/protocol.go | 75 +++++++++++-------- .../syncthing/protocol/protocol_test.go | 6 +- .../syncthing/protocol/wireformat.go | 12 +-- 11 files changed, 154 insertions(+), 92 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/errors.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 238a0231..a3a6f2fa 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -31,7 +31,7 @@ }, { "ImportPath": "github.com/syncthing/protocol", - "Rev": "f9132cae85dcda1caba2f4ba78996d348b00ac6c" + "Rev": "6277c0595c18d42e9db75dfe900463ef093a82d2" }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go index f67fb481..f46b6a8d 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go @@ -13,6 +13,9 @@ type TestModel struct { name string offset int64 size int + hash []byte + flags uint32 + options []Option closedCh chan bool } @@ -22,17 +25,20 @@ func newTestModel() *TestModel { } } -func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { t.folder = folder t.name = name t.offset = offset t.size = size + t.hash = hash + t.flags = flags + t.options = options return t.data, nil } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go b/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go new file mode 100644 index 00000000..31d27af0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go @@ -0,0 +1,51 @@ +// Copyright (C) 2014 The Protocol Authors. + +package protocol + +import ( + "errors" +) + +const ( + ecNoError int32 = iota + ecGeneric + ecNoSuchFile + ecInvalid +) + +var ( + ErrNoError error = nil + ErrGeneric = errors.New("generic error") + ErrNoSuchFile = errors.New("no such file") + ErrInvalid = errors.New("file is invalid") +) + +var lookupError = map[int32]error{ + ecNoError: ErrNoError, + ecGeneric: ErrGeneric, + ecNoSuchFile: ErrNoSuchFile, + ecInvalid: ErrInvalid, +} + +var lookupCode = map[error]int32{ + ErrNoError: ecNoError, + ErrGeneric: ecGeneric, + ErrNoSuchFile: ecNoSuchFile, + ErrInvalid: ecInvalid, +} + +func codeToError(errcode int32) error { + err, ok := lookupError[errcode] + if !ok { + return ErrGeneric + } + return err +} + +func errorToCode(err error) int32 { + code, ok := lookupCode[err] + if !ok { + return ecGeneric + } + return code +} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message.go index 91c33190..c2d89894 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/message.go @@ -1,5 +1,6 @@ // Copyright (C) 2014 The Protocol Authors. +//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -78,8 +79,8 @@ type RequestMessage struct { } type ResponseMessage struct { - Data []byte - Error int32 + Data []byte + Code int32 } type ClusterConfigMessage struct { diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go index 95d72eb1..c179de76 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go @@ -465,13 +465,13 @@ ResponseMessage Structure: \ Data (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Error | +| Code | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct ResponseMessage { opaque Data<>; - int Error; + int Code; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(uint32(o.Error)) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = int32(xr.ReadUint32()) + o.Code = int32(xr.ReadUint32()) return xr.Error() } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go index 5b4b9be6..502a71f2 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go @@ -12,23 +12,23 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.Index(deviceID, folder, files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go index 2fb1654c..21585e30 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go @@ -10,16 +10,16 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { - m.next.Index(deviceID, folder, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { - m.next.IndexUpdate(deviceID, folder, files) +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { - return m.next.Request(deviceID, folder, name, offset, size) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go index d4feea32..072e1278 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go @@ -24,23 +24,30 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) - } - files[i].Name = filepath.FromSlash(f.Name) - } - m.next.Index(deviceID, folder, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + fixupFiles(files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + fixupFiles(files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) +} + +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { + name = filepath.FromSlash(name) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) +} + +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) +} + +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) +} + +func fixupFiles(files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -53,18 +60,4 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI } files[i].Name = filepath.FromSlash(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) -} - -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { - name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, size) -} - -func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { - m.next.ClusterConfig(deviceID, config) -} - -func (m nativeModel) Close(deviceID DeviceID, err error) { - m.next.Close(deviceID, err) } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go index e7b6fe27..b9859203 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go @@ -35,6 +35,7 @@ const ( stateIdxRcvd ) +// FileInfo flags const ( FlagDeleted uint32 = 1 << 12 FlagInvalid = 1 << 13 @@ -48,6 +49,17 @@ const ( SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) +// IndexMessage message flags (for IndexUpdate) +const ( + FlagIndexTemporary uint32 = 1 << iota +) + +// Request message flags +const ( + FlagRequestTemporary uint32 = 1 << iota +) + +// ClusterConfigMessage.Folders.Devices flags const ( FlagShareTrusted uint32 = 1 << 0 FlagShareReadOnly = 1 << 1 @@ -66,11 +78,11 @@ type pongMessage struct{ EmptyMessage } type Model interface { // An index was received from the peer device - Index(deviceID DeviceID, folder string, files []FileInfo) + Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // An index update was received from the peer device - IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) + IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) + Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) // A cluster configuration message was received ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) // The peer device closed the connection @@ -80,9 +92,9 @@ type Model interface { type Connection interface { ID() DeviceID Name() string - Index(folder string, files []FileInfo) error - IndexUpdate(folder string, files []FileInfo) error - Request(folder string, name string, offset int64, size int) ([]byte, error) + Index(folder string, files []FileInfo, flags uint32, options []Option) error + IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error + Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics } @@ -169,7 +181,7 @@ func (c *rawConnection) Name() string { } // Index writes the list of file information to the connected peer device -func (c *rawConnection) Index(folder string, idx []FileInfo) error { +func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -177,15 +189,17 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndex, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil } // IndexUpdate writes the list of file information to the connected peer device as an update -func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { +func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -193,15 +207,17 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndexUpdate, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil } // Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) { +func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { var id int select { case id = <-c.nextID: @@ -218,10 +234,13 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i c.awaitingMut.Unlock() ok := c.send(id, messageTypeRequest, RequestMessage{ - Folder: folder, - Name: name, - Offset: offset, - Size: int32(size), + Folder: folder, + Name: name, + Offset: offset, + Size: int32(size), + Hash: hash, + Flags: flags, + Options: options, }) if !ok { return nil, ErrClosed @@ -280,11 +299,6 @@ func (c *rawConnection) readerLoop() (err error) { switch msg := msg.(type) { case IndexMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags) - } - switch hdr.msgType { case messageTypeIndex: if c.state < stateCCRcvd { @@ -301,10 +315,6 @@ func (c *rawConnection) readerLoop() (err error) { } case RequestMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags) - } if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } @@ -460,16 +470,16 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { func (c *rawConnection) handleIndex(im IndexMessage) { if debug { - l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func filterIndexMessageFiles(fs []FileInfo) []FileInfo { @@ -499,10 +509,11 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) + data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) c.send(msgID, messageTypeResponse, ResponseMessage{ Data: data, + Code: errorToCode(err), }) } @@ -510,7 +521,7 @@ func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { c.awaitingMut.Lock() if rc := c.awaiting[msgID]; rc != nil { c.awaiting[msgID] = nil - rc <- asyncResult{resp.Data, nil} + rc <- asyncResult{resp.Data, codeToError(resp.Code)} close(rc) } c.awaitingMut.Unlock() diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go index c1048cdc..3ff1042c 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go @@ -229,10 +229,10 @@ func TestClose(t *testing.T) { t.Error("Ping should not return true") } - c0.Index("default", nil) - c0.Index("default", nil) + c0.Index("default", nil, 0, nil) + c0.Index("default", nil, 0, nil) - if _, err := c0.Request("default", "foo", 0, 0); err == nil { + if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { t.Error("Request should return an error") } } diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go b/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go index 4eab3d37..9411955b 100644 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go +++ b/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go @@ -20,7 +20,7 @@ func (c wireFormatConnection) Name() string { return c.next.Name() } -func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { +func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -28,10 +28,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.Index(folder, myFs) + return c.next.Index(folder, myFs, flags, options) } -func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { +func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -39,12 +39,12 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.IndexUpdate(folder, myFs) + return c.next.IndexUpdate(folder, myFs, flags, options) } -func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { +func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(folder, name, offset, size) + return c.next.Request(folder, name, offset, size, hash, flags, options) } func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { From 489e2e6ad587ec8ef11d6f2ce07e718abfe84c00 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 14 Jan 2015 22:11:31 +0000 Subject: [PATCH 2/5] Update Model function signatures --- cmd/syncthing/main.go | 2 +- internal/model/model.go | 20 ++++++------- internal/model/model_test.go | 54 ++++++++++++++++++------------------ internal/model/rwfolder.go | 2 +- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index dde7b27d..e6b56d06 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -533,7 +533,7 @@ func syncthingMain() { if device == myID { continue } - m.Index(device, folderCfg.ID, nil) + m.Index(device, folderCfg.ID, nil, 0, nil) } } diff --git a/internal/model/model.go b/internal/model/model.go index daf564f5..ab23b5bf 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -423,7 +423,7 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]db.FileInfoTruncated, // Index is called when a new device is connected and we receive their full index. // Implements the protocol.Model interface. -func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) { +func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) { if debug { l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs)) } @@ -475,7 +475,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F // IndexUpdate is called for incremental updates to connected devices' indexes. // Implements the protocol.Model interface. -func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) { +func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) { if debug { l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs)) } @@ -672,7 +672,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) { // Request returns the specified data segment by reading it from local disk. // Implements the protocol.Model interface. -func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int) ([]byte, error) { +func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) { if offset < 0 || size < 0 { return nil, ErrNoSuchFile } @@ -975,7 +975,7 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize { if initial { - if err = conn.Index(folder, batch); err != nil { + if err = conn.Index(folder, batch, 0, nil); err != nil { return false } if debug { @@ -983,7 +983,7 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold } initial = false } else { - if err = conn.IndexUpdate(folder, batch); err != nil { + if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil { return false } if debug { @@ -1001,12 +1001,12 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold }) if initial && err == nil { - err = conn.Index(folder, batch) + err = conn.Index(folder, batch, 0, nil) if debug && err == nil { l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch)) } } else if len(batch) > 0 && err == nil { - err = conn.IndexUpdate(folder, batch) + err = conn.IndexUpdate(folder, batch, 0, nil) if debug && err == nil { l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch)) } @@ -1029,7 +1029,7 @@ func (m *Model) updateLocal(folder string, f protocol.FileInfo) { }) } -func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte) ([]byte, error) { +func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) { m.pmut.RLock() nc, ok := m.protoConn[deviceID] m.pmut.RUnlock() @@ -1039,10 +1039,10 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o } if debug { - l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x", m, deviceID, folder, name, offset, size, hash) + l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x f=%x op=%s", m, deviceID, folder, name, offset, size, hash, flags, options) } - return nc.Request(folder, name, offset, size) + return nc.Request(folder, name, offset, size, hash, flags, options) } func (m *Model) AddFolder(cfg config.FolderConfiguration) { diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 46562b66..6672cb06 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -94,7 +94,7 @@ func TestRequest(t *testing.T) { m.ScanFolder("default") // Existing, shared file - bs, err := m.Request(device1, "default", "foo", 0, 6) + bs, err := m.Request(device1, "default", "foo", 0, 6, nil, 0, nil) if err != nil { t.Error(err) } @@ -103,7 +103,7 @@ func TestRequest(t *testing.T) { } // Existing, nonshared file - bs, err = m.Request(device2, "default", "foo", 0, 6) + bs, err = m.Request(device2, "default", "foo", 0, 6, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -112,7 +112,7 @@ func TestRequest(t *testing.T) { } // Nonexistent file - bs, err = m.Request(device1, "default", "nonexistent", 0, 6) + bs, err = m.Request(device1, "default", "nonexistent", 0, 6, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -121,7 +121,7 @@ func TestRequest(t *testing.T) { } // Shared folder, but disallowed file name - bs, err = m.Request(device1, "default", "../walk.go", 0, 6) + bs, err = m.Request(device1, "default", "../walk.go", 0, 6, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -130,7 +130,7 @@ func TestRequest(t *testing.T) { } // Larger block than available - bs, err = m.Request(device1, "default", "foo", 0, 42) + bs, err = m.Request(device1, "default", "foo", 0, 42, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -139,7 +139,7 @@ func TestRequest(t *testing.T) { } // Negative offset - bs, err = m.Request(device1, "default", "foo", -4, 6) + bs, err = m.Request(device1, "default", "foo", -4, 6, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -148,7 +148,7 @@ func TestRequest(t *testing.T) { } // Negative size - bs, err = m.Request(device1, "default", "foo", 4, -4) + bs, err = m.Request(device1, "default", "foo", 4, -4, nil, 0, nil) if err == nil { t.Error("Unexpected nil error on insecure file read") } @@ -180,7 +180,7 @@ func BenchmarkIndex10000(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) } } @@ -193,7 +193,7 @@ func BenchmarkIndex00100(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) } } @@ -203,11 +203,11 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) { m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { - m.IndexUpdate(device1, "default", files) + m.IndexUpdate(device1, "default", files, 0, nil) } } @@ -217,12 +217,12 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) { m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) ufiles := genFiles(100) b.ResetTimer() for i := 0; i < b.N; i++ { - m.IndexUpdate(device1, "default", ufiles) + m.IndexUpdate(device1, "default", ufiles, 0, nil) } } @@ -232,12 +232,12 @@ func BenchmarkIndexUpdate10000f00001(b *testing.B) { m.AddFolder(defaultFolderConfig) m.ScanFolder("default") files := genFiles(10000) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) ufiles := genFiles(1) b.ResetTimer() for i := 0; i < b.N; i++ { - m.IndexUpdate(device1, "default", ufiles) + m.IndexUpdate(device1, "default", ufiles, 0, nil) } } @@ -262,15 +262,15 @@ func (f FakeConnection) Option(string) string { return "" } -func (FakeConnection) Index(string, []protocol.FileInfo) error { +func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error { return nil } -func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error { +func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error { return nil } -func (f FakeConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { +func (f FakeConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) { return f.requestData, nil } @@ -306,11 +306,11 @@ func BenchmarkRequest(b *testing.B) { requestData: []byte("some data to return"), } m.AddConnection(fc, fc) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { - data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil) + data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, nil) if err != nil { b.Error(err) } @@ -564,7 +564,7 @@ func TestRefuseUnknownBits(t *testing.T) { Name: "valid", Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink), }, - }) + }, 0, nil) for _, name := range []string{"invalid1", "invalid2", "invalid3"} { f, ok := m.CurrentGlobalFile("default", name) @@ -666,7 +666,7 @@ func TestGlobalDirectoryTree(t *testing.T) { return string(bytes) } - m.Index(device1, "default", testdata) + m.Index(device1, "default", testdata, 0, nil) result := m.GlobalDirectoryTree("default", "", -1, false) @@ -925,7 +925,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) { return string(bytes) } - m.Index(device1, "default", testdata) + m.Index(device1, "default", testdata, 0, nil) result := m.GlobalDirectoryTree("default", "", -1, false) @@ -996,7 +996,7 @@ func BenchmarkTree_10000_50(b *testing.B) { m.ScanFolder("default") files := genDeepFiles(10000, 50) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1011,7 +1011,7 @@ func BenchmarkTree_10000_10(b *testing.B) { m.ScanFolder("default") files := genDeepFiles(10000, 10) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1026,7 +1026,7 @@ func BenchmarkTree_00100_50(b *testing.B) { m.ScanFolder("default") files := genDeepFiles(100, 50) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1041,7 +1041,7 @@ func BenchmarkTree_00100_10(b *testing.B) { m.ScanFolder("default") files := genDeepFiles(100, 10) - m.Index(device1, "default", files) + m.Index(device1, "default", files, 0, nil) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/internal/model/rwfolder.go b/internal/model/rwfolder.go index b6c15320..332b7bec 100644 --- a/internal/model/rwfolder.go +++ b/internal/model/rwfolder.go @@ -918,7 +918,7 @@ func (p *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul // Fetch the block, while marking the selected device as in use so that // leastBusy can select another device when someone else asks. activity.using(selected) - buf, lastError := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash) + buf, lastError := p.model.requestGlobal(selected, p.folder, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, 0, nil) activity.done(selected) if lastError != nil { continue From 0e341832e0cc91b6fb396b108a61dce810f3555c Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 14 Jan 2015 22:28:19 +0000 Subject: [PATCH 3/5] Handle unknown flags at the model --- internal/model/model.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/model/model.go b/internal/model/model.go index ab23b5bf..2414e47a 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -424,6 +424,11 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]db.FileInfoTruncated, // Index is called when a new device is connected and we receive their full index. // Implements the protocol.Model interface. func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) { + if flags != 0 { + l.Warnln("protocol error: unknown flags 0x%x in Index message", flags) + return + } + if debug { l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs)) } @@ -476,6 +481,11 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F // IndexUpdate is called for incremental updates to connected devices' indexes. // Implements the protocol.Model interface. func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) { + if flags != 0 { + l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags) + return + } + if debug { l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs)) } @@ -682,6 +692,11 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset return nil, ErrNoSuchFile } + if flags != 0 { + // We don't currently support or expect any flags. + return nil, fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags) + } + // Verify that the requested file exists in the local model. m.fmut.RLock() folderFiles, ok := m.folderFiles[folder] From c12265499ad03549dd5fcb88c8e7925a488c39ef Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 14 Jan 2015 22:41:25 +0000 Subject: [PATCH 4/5] Response errors may be protocol defined errors --- internal/model/model.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal/model/model.go b/internal/model/model.go index 2414e47a..a756f5ec 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -85,9 +85,6 @@ type Model struct { } var ( - ErrNoSuchFile = errors.New("no such file") - ErrInvalid = errors.New("file is invalid") - SymlinkWarning = sync.Once{} ) @@ -684,12 +681,12 @@ func (m *Model) Close(device protocol.DeviceID, err error) { // Implements the protocol.Model interface. func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []protocol.Option) ([]byte, error) { if offset < 0 || size < 0 { - return nil, ErrNoSuchFile + return nil, protocol.ErrNoSuchFile } if !m.folderSharedWith(folder, deviceID) { l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder) - return nil, ErrNoSuchFile + return nil, protocol.ErrNoSuchFile } if flags != 0 { @@ -704,26 +701,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset if !ok { l.Warnf("Request from %s for file %s in nonexistent folder %q", deviceID, name, folder) - return nil, ErrNoSuchFile + return nil, protocol.ErrNoSuchFile } lf, ok := folderFiles.Get(protocol.LocalDeviceID, name) if !ok { - return nil, ErrNoSuchFile + return nil, protocol.ErrNoSuchFile } if lf.IsInvalid() || lf.IsDeleted() { if debug { l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, size, lf) } - return nil, ErrInvalid + return nil, protocol.ErrInvalid } if offset > lf.Size() { if debug { l.Debugf("%v REQ(in; nonexistent): %s: %q o=%d s=%d", m, deviceID, name, offset, size) } - return nil, ErrNoSuchFile + return nil, protocol.ErrNoSuchFile } if debug && deviceID != protocol.LocalDeviceID { From 0401a075076968778044cde68b6d6f58072a2751 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 14 Jan 2015 23:00:00 +0000 Subject: [PATCH 5/5] Change existingBlocks map type --- internal/model/rwfolder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/model/rwfolder.go b/internal/model/rwfolder.go index 332b7bec..983eed39 100644 --- a/internal/model/rwfolder.go +++ b/internal/model/rwfolder.go @@ -708,9 +708,9 @@ func (p *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks) // block.String() returns a string unique to the block - existingBlocks := make(map[string]bool, len(tempCopyBlocks)) + existingBlocks := make(map[string]struct{}, len(tempCopyBlocks)) for _, block := range tempCopyBlocks { - existingBlocks[block.String()] = true + existingBlocks[block.String()] = struct{}{} } // Since the blocks are already there, we don't need to get them.