perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add Local Version field to files, send index in segments.

Browse Source
This commit is contained in:
Jakob Borg
2014-07-15 13:04:37 +02:00
parent fccdd85cc1
commit 8b349945de
16 changed files with 291 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
protocol/PROTOCOL.md
View File

@@ -182,7 +182,7 @@ Cluster Config messages MUST NOT be sent after the initial exchange.
| Flags |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Max Version (64 bits) +
+ Max Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -255,13 +255,13 @@ The Node Flags field contains the following single bit flags:
Exactly one of the T, R or S bits MUST be set.
The Node Max Version field contains the highest file version number of
the files already known to be in the index sent by this node. If nothing
is known about the index of a given node, this field MUST be set to
zero. When receiving a Cluster Config message with a non-zero Max
The Node Max Local Version field contains the highest local file version
number of the files already known to be in the index sent by this node.
If nothing is known about the index of a given node, this field MUST be
set to zero. When receiving a Cluster Config message with a non-zero Max
Version for the local node ID, a node MAY elect to send an Index Update
message containing only files with higher version numbers in place of
the initial Index message.
message containing only files with higher local version numbers in place
of the initial Index message.
The Options field contain option values to be used in an implementation
specific manner. The options list is conceptually a map of Key => Value
@@ -292,7 +292,7 @@ peers acting in a specific manner as a result of sent options.
struct Node {
string ID<>;
unsigned int Flags;
unsigned hyper MaxVersion;
unsigned hyper MaxLocalVersion;
}
struct Option {
@@ -359,6 +359,10 @@ Index message MUST be sent. There is no response to the Index message.
+ Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Blocks |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
@@ -400,6 +404,10 @@ detected and received change. The combination of Repository, Name and
Version uniquely identifies the contents of a file at a given point in
time.
The Local Version field is the value of a node local monotonic clock at
the time of last local database update to a file. The clock ticks on
every local database update.
The Flags field is made up of the following single bit flags:
0 1 2 3
@@ -458,6 +466,7 @@ block which may represent a smaller amount of data.
unsigned int Flags;
hyper Modified;
unsigned hyper Version;
unsigned hyper LocalVer;
BlockInfo Blocks<>;
}

17
protocol/message.go
View File

@@ -12,11 +12,12 @@ type IndexMessage struct {
}
type FileInfo struct {
Name string // max:1024
Flags uint32
Modified int64
Version uint64
Blocks []BlockInfo // max:1000000
Name string // max:1024
Flags uint32
Modified int64
Version uint64
LocalVersion uint64
Blocks []BlockInfo // max:1000000
}
func (f FileInfo) String() string {
@@ -69,9 +70,9 @@ type Repository struct {
}
type Node struct {
ID []byte // max:32
Flags uint32
MaxVersion uint64
ID []byte // max:32
Flags uint32
MaxLocalVersion uint64
}
type Option struct {

15
protocol/message_xdr.go
View File

@@ -121,6 +121,10 @@ FileInfo Structure:
+ Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Blocks |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
@@ -134,6 +138,7 @@ struct FileInfo {
unsigned int Flags;
hyper Modified;
unsigned hyper Version;
unsigned hyper LocalVersion;
BlockInfo Blocks<1000000>;
}
@@ -163,6 +168,7 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteUint32(o.Flags)
xw.WriteUint64(uint64(o.Modified))
xw.WriteUint64(o.Version)
xw.WriteUint64(o.LocalVersion)
if len(o.Blocks) > 1000000 {
return xw.Tot(), xdr.ErrElementSizeExceeded
}
@@ -189,6 +195,7 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
o.Flags = xr.ReadUint32()
o.Modified = int64(xr.ReadUint64())
o.Version = xr.ReadUint64()
o.LocalVersion = xr.ReadUint64()
_BlocksSize := int(xr.ReadUint32())
if _BlocksSize > 1000000 {
return xdr.ErrElementSizeExceeded
@@ -567,7 +574,7 @@ Node Structure:
| Flags |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Max Version (64 bits) +
+ Max Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -575,7 +582,7 @@ Node Structure:
struct Node {
opaque ID<32>;
unsigned int Flags;
unsigned hyper MaxVersion;
unsigned hyper MaxLocalVersion;
}
*/
@@ -602,7 +609,7 @@ func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
}
xw.WriteBytes(o.ID)
xw.WriteUint32(o.Flags)
xw.WriteUint64(o.MaxVersion)
xw.WriteUint64(o.MaxLocalVersion)
return xw.Tot(), xw.Error()
}
@@ -620,7 +627,7 @@ func (o *Node) UnmarshalXDR(bs []byte) error {
func (o *Node) decodeXDR(xr *xdr.Reader) error {
o.ID = xr.ReadBytesMax(32)
o.Flags = xr.ReadUint32()
o.MaxVersion = xr.ReadUint64()
o.MaxLocalVersion = xr.ReadUint64()
return xr.Error()
}

70
protocol/protocol.go
View File

@@ -66,7 +66,9 @@ type Model interface {
type Connection interface {
ID() NodeID
Index(repo string, files []FileInfo)
Name() string
Index(repo string, files []FileInfo) error
IndexUpdate(repo string, files []FileInfo) error
Request(repo string, name string, offset int64, size int) ([]byte, error)
ClusterConfig(config ClusterConfigMessage)
Statistics() Statistics
@@ -74,6 +76,7 @@ type Connection interface {
type rawConnection struct {
id NodeID
name string
receiver Model
state int
@@ -87,8 +90,7 @@ type rawConnection struct {
awaiting []chan asyncResult
awaitingMut sync.Mutex
idxSent map[string]map[string]uint64
idxMut sync.Mutex // ensures serialization of Index calls
idxMut sync.Mutex // ensures serialization of Index calls
nextID chan int
outbox chan []encodable
@@ -106,15 +108,16 @@ const (
pingIdleTime = 60 * time.Second
)
func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model) Connection {
func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string) Connection {
cr := &countingReader{Reader: reader}
cw := &countingWriter{Writer: writer}
rb := bufio.NewReader(cr)
wb := bufio.NewWriter(cw)
wb := bufio.NewWriterSize(cw, 65536)
c := rawConnection{
id: nodeID,
name: name,
receiver: nativeModel{receiver},
state: stateInitial,
cr: cr,
@@ -123,7 +126,6 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M
wb: wb,
xw: xdr.NewWriter(wb),
awaiting: make([]chan asyncResult, 0x1000),
idxSent: make(map[string]map[string]uint64),
outbox: make(chan []encodable),
nextID: make(chan int),
closed: make(chan struct{}),
@@ -142,36 +144,34 @@ func (c *rawConnection) ID() NodeID {
return c.id
}
func (c *rawConnection) Name() string {
return c.name
}
// Index writes the list of file information to the connected peer node
func (c *rawConnection) Index(repo string, idx []FileInfo) {
func (c *rawConnection) Index(repo string, idx []FileInfo) error {
select {
case <-c.closed:
return ErrClosed
default:
}
c.idxMut.Lock()
defer c.idxMut.Unlock()
c.send(header{0, -1, messageTypeIndex}, IndexMessage{repo, idx})
c.idxMut.Unlock()
return nil
}
var msgType int
if c.idxSent[repo] == nil {
// This is the first time we send an index.
msgType = messageTypeIndex
c.idxSent[repo] = make(map[string]uint64)
for _, f := range idx {
c.idxSent[repo][f.Name] = f.Version
}
} else {
// We have sent one full index. Only send updates now.
msgType = messageTypeIndexUpdate
var diff []FileInfo
for _, f := range idx {
if vs, ok := c.idxSent[repo][f.Name]; !ok || f.Version != vs {
diff = append(diff, f)
c.idxSent[repo][f.Name] = f.Version
}
}
idx = diff
}
if msgType == messageTypeIndex || len(idx) > 0 {
c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
// IndexUpdate writes the list of file information to the connected peer node as an update
func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error {
select {
case <-c.closed:
return ErrClosed
default:
}
c.idxMut.Lock()
c.send(header{0, -1, messageTypeIndexUpdate}, IndexMessage{repo, idx})
c.idxMut.Unlock()
return nil
}
// Request returns the bytes for the specified block after fetching them from the connected peer.
@@ -445,15 +445,13 @@ func (c *rawConnection) send(h header, es ...encodable) bool {
}
func (c *rawConnection) writerLoop() {
var err error
for {
select {
case es := <-c.outbox:
for _, e := range es {
e.encodeXDR(c.xw)
}
if err = c.flush(); err != nil {
if err := c.flush(); err != nil {
c.close(err)
return
}
@@ -471,11 +469,9 @@ func (c *rawConnection) flush() error {
if err := c.xw.Error(); err != nil {
return err
}
if err := c.wb.Flush(); err != nil {
return err
}
return nil
}

19
protocol/wireformat.go
View File

@@ -18,7 +18,11 @@ func (c wireFormatConnection) ID() NodeID {
return c.next.ID()
}
func (c wireFormatConnection) Index(repo string, fs []FileInfo) {
func (c wireFormatConnection) Name() string {
return c.next.Name()
}
func (c wireFormatConnection) Index(repo string, fs []FileInfo) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
@@ -26,7 +30,18 @@ func (c wireFormatConnection) Index(repo string, fs []FileInfo) {
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
c.next.Index(repo, myFs)
return c.next.Index(repo, myFs)
}
func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
for i := range fs {
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
return c.next.IndexUpdate(repo, myFs)
}
func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) {