perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model, lib/protocol: Implement temporary indexes (fixes #950)

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/2252
This commit is contained in:
AudriusButkevicius
2016-04-15 10:59:41 +00:00
committed by Jakob Borg
parent a4cd4cc253
commit 1a5f524ae4
28 changed files with 1612 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/protocol/common_test.go
View File

@@ -49,6 +49,9 @@ func (t *TestModel) Close(deviceID DeviceID, err error) {
func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
}
func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate, uint32, []Option) {
}
func (t *TestModel) closedError() error {
select {
case <-t.closedCh:

14
lib/protocol/message.go
View File

@@ -138,6 +138,13 @@ type ClusterConfigMessage struct {
Options []Option // max:64
}
type DownloadProgressMessage struct {
Folder string // max:64
Updates []FileDownloadProgressUpdate // max:1000000
Flags uint32
Options []Option // max:64
}
func (o *ClusterConfigMessage) GetOption(key string) string {
for _, option := range o.Options {
if option.Key == key {
@@ -166,6 +173,13 @@ type Device struct {
Options []Option // max:64
}
type FileDownloadProgressUpdate struct {
UpdateType uint32
Name string // max:8192
Version Vector
BlockIndexes []int32 // max:1000000
}
type Option struct {
Key string // max:64
Value string // max:1024

232
lib/protocol/message_xdr.go
View File

@@ -690,6 +690,135 @@ func (o *ClusterConfigMessage) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
/*
DownloadProgressMessage Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Folder (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Updates |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more FileDownloadProgressUpdate Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Flags |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Options |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more Option Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct DownloadProgressMessage {
string Folder<64>;
FileDownloadProgressUpdate Updates<1000000>;
unsigned int Flags;
Option Options<64>;
}
*/
func (o DownloadProgressMessage) XDRSize() int {
return 4 + len(o.Folder) + xdr.Padding(len(o.Folder)) +
4 + xdr.SizeOfSlice(o.Updates) + 4 +
4 + xdr.SizeOfSlice(o.Options)
}
func (o DownloadProgressMessage) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o DownloadProgressMessage) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o DownloadProgressMessage) MarshalXDRInto(m *xdr.Marshaller) error {
if l := len(o.Folder); l > 64 {
return xdr.ElementSizeExceeded("Folder", l, 64)
}
m.MarshalString(o.Folder)
if l := len(o.Updates); l > 1000000 {
return xdr.ElementSizeExceeded("Updates", l, 1000000)
}
m.MarshalUint32(uint32(len(o.Updates)))
for i := range o.Updates {
if err := o.Updates[i].MarshalXDRInto(m); err != nil {
return err
}
}
m.MarshalUint32(o.Flags)
if l := len(o.Options); l > 64 {
return xdr.ElementSizeExceeded("Options", l, 64)
}
m.MarshalUint32(uint32(len(o.Options)))
for i := range o.Options {
if err := o.Options[i].MarshalXDRInto(m); err != nil {
return err
}
}
return m.Error
}
func (o *DownloadProgressMessage) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *DownloadProgressMessage) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Folder = u.UnmarshalStringMax(64)
_UpdatesSize := int(u.UnmarshalUint32())
if _UpdatesSize < 0 {
return xdr.ElementSizeExceeded("Updates", _UpdatesSize, 1000000)
} else if _UpdatesSize == 0 {
o.Updates = nil
} else {
if _UpdatesSize > 1000000 {
return xdr.ElementSizeExceeded("Updates", _UpdatesSize, 1000000)
}
if _UpdatesSize <= len(o.Updates) {
o.Updates = o.Updates[:_UpdatesSize]
} else {
o.Updates = make([]FileDownloadProgressUpdate, _UpdatesSize)
}
for i := range o.Updates {
(&o.Updates[i]).UnmarshalXDRFrom(u)
}
}
o.Flags = u.UnmarshalUint32()
_OptionsSize := int(u.UnmarshalUint32())
if _OptionsSize < 0 {
return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
} else if _OptionsSize == 0 {
o.Options = nil
} else {
if _OptionsSize > 64 {
return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
}
if _OptionsSize <= len(o.Options) {
o.Options = o.Options[:_OptionsSize]
} else {
o.Options = make([]Option, _OptionsSize)
}
for i := range o.Options {
(&o.Options[i]).UnmarshalXDRFrom(u)
}
}
return u.Error
}
/*
Folder Structure:
0 1 2 3
@@ -996,6 +1125,109 @@ func (o *Device) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
/*
FileDownloadProgressUpdate Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Update Type |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Name (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Vector Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Block Indexes |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
| Block Indexes (n items) |
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct FileDownloadProgressUpdate {
unsigned int UpdateType;
string Name<8192>;
Vector Version;
int BlockIndexes<1000000>;
}
*/
func (o FileDownloadProgressUpdate) XDRSize() int {
return 4 +
4 + len(o.Name) + xdr.Padding(len(o.Name)) +
o.Version.XDRSize() +
4 + len(o.BlockIndexes)*4
}
func (o FileDownloadProgressUpdate) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o FileDownloadProgressUpdate) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o FileDownloadProgressUpdate) MarshalXDRInto(m *xdr.Marshaller) error {
m.MarshalUint32(o.UpdateType)
if l := len(o.Name); l > 8192 {
return xdr.ElementSizeExceeded("Name", l, 8192)
}
m.MarshalString(o.Name)
if err := o.Version.MarshalXDRInto(m); err != nil {
return err
}
if l := len(o.BlockIndexes); l > 1000000 {
return xdr.ElementSizeExceeded("BlockIndexes", l, 1000000)
}
m.MarshalUint32(uint32(len(o.BlockIndexes)))
for i := range o.BlockIndexes {
m.MarshalUint32(uint32(o.BlockIndexes[i]))
}
return m.Error
}
func (o *FileDownloadProgressUpdate) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *FileDownloadProgressUpdate) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.UpdateType = u.UnmarshalUint32()
o.Name = u.UnmarshalStringMax(8192)
(&o.Version).UnmarshalXDRFrom(u)
_BlockIndexesSize := int(u.UnmarshalUint32())
if _BlockIndexesSize < 0 {
return xdr.ElementSizeExceeded("BlockIndexes", _BlockIndexesSize, 1000000)
} else if _BlockIndexesSize == 0 {
o.BlockIndexes = nil
} else {
if _BlockIndexesSize > 1000000 {
return xdr.ElementSizeExceeded("BlockIndexes", _BlockIndexesSize, 1000000)
}
if _BlockIndexesSize <= len(o.BlockIndexes) {
o.BlockIndexes = o.BlockIndexes[:_BlockIndexesSize]
} else {
o.BlockIndexes = make([]int32, _BlockIndexesSize)
}
for i := range o.BlockIndexes {
o.BlockIndexes[i] = int32(u.UnmarshalUint32())
}
}
return u.Error
}
/*
Option Structure:
0 1 2 3

16
lib/protocol/nativemodel_darwin.go
View File

@@ -9,32 +9,24 @@ package protocol
import "golang.org/x/text/unicode/norm"
type nativeModel struct {
next Model
Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.next.Index(deviceID, folder, files, flags, options)
m.Model.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.next.IndexUpdate(deviceID, folder, files, flags, options)
m.Model.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
name = norm.NFD.String(name)
return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
}

22
lib/protocol/nativemodel_unix.go
View File

@@ -7,25 +7,5 @@ package protocol
// Normal Unixes uses NFC and slashes, which is the wire format.
type nativeModel struct {
next Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
m.next.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
m.next.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
Model
}

16
lib/protocol/nativemodel_windows.go
View File

@@ -21,30 +21,22 @@ var disallowedCharacters = string([]rune{
})
type nativeModel struct {
next Model
Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
fixupFiles(folder, files)
m.next.Index(deviceID, folder, files, flags, options)
m.Model.Index(deviceID, folder, files, flags, options)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
fixupFiles(folder, files)
m.next.IndexUpdate(deviceID, folder, files, flags, options)
m.Model.IndexUpdate(deviceID, folder, files, flags, options)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
name = filepath.FromSlash(name)
return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
}
func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
m.next.ClusterConfig(deviceID, config)
}
func (m nativeModel) Close(deviceID DeviceID, err error) {
m.next.Close(deviceID, err)
return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
}
func fixupFiles(folder string, files []FileInfo) {

81
lib/protocol/protocol.go
View File

@@ -24,13 +24,14 @@ const (
)
const (
messageTypeClusterConfig = 0
messageTypeIndex = 1
messageTypeRequest = 2
messageTypeResponse = 3
messageTypePing = 4
messageTypeIndexUpdate = 6
messageTypeClose = 7
messageTypeClusterConfig = 0
messageTypeIndex = 1
messageTypeRequest = 2
messageTypeResponse = 3
messageTypePing = 4
messageTypeIndexUpdate = 6
messageTypeClose = 7
messageTypeDownloadProgress = 8
)
const (
@@ -52,22 +53,29 @@ const (
SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
)
// IndexMessage message flags (for IndexUpdate)
const (
FlagIndexTemporary uint32 = 1 << iota
)
// Request message flags
const (
FlagRequestTemporary uint32 = 1 << iota
FlagFromTemporary uint32 = 1 << iota
)
// FileDownloadProgressUpdate update types
const (
UpdateTypeAppend uint32 = iota
UpdateTypeForget
)
// CLusterConfig flags
const (
FlagClusterConfigTemporaryIndexes uint32 = 1 << 0
)
// ClusterConfigMessage.Folders flags
const (
FlagFolderReadOnly uint32 = 1 << 0
FlagFolderIgnorePerms = 1 << 1
FlagFolderIgnoreDelete = 1 << 2
FlagFolderAll = 1<<3 - 1
FlagFolderReadOnly uint32 = 1 << 0
FlagFolderIgnorePerms = 1 << 1
FlagFolderIgnoreDelete = 1 << 2
FlagFolderDisabledTempIndexes = 1 << 3
FlagFolderAll = 1<<4 - 1
)
// ClusterConfigMessage.Folders.Devices flags
@@ -97,6 +105,8 @@ type Model interface {
ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
// The peer device closed the connection
Close(deviceID DeviceID, err error)
// The peer device sent progress updates for the files it is currently downloading
DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option)
}
type Connection interface {
@@ -105,8 +115,9 @@ type Connection interface {
Name() string
Index(folder string, files []FileInfo, flags uint32, options []Option) error
IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error
Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
ClusterConfig(config ClusterConfigMessage)
DownloadProgress(folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option)
Statistics() Statistics
Closed() bool
}
@@ -242,7 +253,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32,
}
// Request returns the bytes for the specified block after fetching them from the connected peer.
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
var id int
select {
case id = <-c.nextID:
@@ -250,6 +261,12 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
return nil, ErrClosed
}
var flags uint32
if fromTemporary {
flags |= FlagFromTemporary
}
c.awaitingMut.Lock()
if ch := c.awaiting[id]; ch != nil {
panic("id taken")
@@ -265,7 +282,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
Size: int32(size),
Hash: hash,
Flags: flags,
Options: options,
Options: nil,
}, nil)
if !ok {
return nil, ErrClosed
@@ -292,6 +309,16 @@ func (c *rawConnection) Closed() bool {
}
}
// DownloadProgress sends the progress updates for the files that are currently being downloaded.
func (c *rawConnection) DownloadProgress(folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option) {
c.send(-1, messageTypeDownloadProgress, DownloadProgressMessage{
Folder: folder,
Updates: updates,
Flags: flags,
Options: options,
}, nil)
}
func (c *rawConnection) ping() bool {
var id int
select {
@@ -359,6 +386,12 @@ func (c *rawConnection) readerLoop() (err error) {
}
c.handleResponse(hdr.msgID, msg)
case DownloadProgressMessage:
if state != stateReady {
return fmt.Errorf("protocol error: response message in state %d", state)
}
c.receiver.DownloadProgress(c.id, msg.Folder, msg.Updates, msg.Flags, msg.Options)
case pingMessage:
if state != stateReady {
return fmt.Errorf("protocol error: ping message in state %d", state)
@@ -469,6 +502,14 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
err = cm.UnmarshalXDR(msgBuf)
msg = cm
case messageTypeDownloadProgress:
var dp DownloadProgressMessage
err := dp.UnmarshalXDR(msgBuf)
if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
err = nil
}
msg = dp
default:
err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
}

2
lib/protocol/protocol_test.go
View File

@@ -183,7 +183,7 @@ func TestClose(t *testing.T) {
c0.Index("default", nil, 0, nil)
c0.Index("default", nil, 0, nil)
if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
t.Error("Request should return an error")
}
}

4
lib/protocol/wireformat.go
View File

@@ -34,7 +34,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags ui
return c.Connection.IndexUpdate(folder, myFs, flags, options)
}
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
name = norm.NFC.String(filepath.ToSlash(name))
return c.Connection.Request(folder, name, offset, size, hash, flags, options)
return c.Connection.Request(folder, name, offset, size, hash, fromTemporary)
}