perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactor compression support, now at message level.

Browse Source
This commit is contained in:
Jakob Borg
2014-07-28 11:31:22 +02:00
parent 6a441d5013
commit 6c5c14f35f
8 changed files with 418 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

363
protocol/protocol.go
View File

@@ -6,16 +6,21 @@ package protocol
import (
"bufio"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/calmh/syncthing/xdr"
lz4 "github.com/bkaradzic/go-lz4"
)
const BlockSize = 128 * 1024
const (
BlockSize = 128 * 1024
MinCompressedSize = 128 // message must be this big to enable compression
)
const (
messageTypeClusterConfig = 0
@@ -82,21 +87,22 @@ type rawConnection struct {
state int
cr *countingReader
xr *xdr.Reader
cw *countingWriter
wb *bufio.Writer
xw *xdr.Writer
awaiting []chan asyncResult
awaiting [4096]chan asyncResult
awaitingMut sync.Mutex
idxMut sync.Mutex // ensures serialization of Index calls
nextID chan int
outbox chan []encodable
outbox chan hdrMsg
closed chan struct{}
once sync.Once
rdbuf0 []byte // used & reused by readMessage
rdbuf1 []byte // used & reused by readMessage
}
type asyncResult struct {
@@ -104,36 +110,32 @@ type asyncResult struct {
err error
}
type hdrMsg struct {
hdr header
msg encodable
}
type encodable interface {
AppendXDR([]byte) []byte
}
const (
pingTimeout = 30 * time.Second
pingIdleTime = 60 * time.Second
)
func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string) Connection {
// Byte counters are at the lowest level, counting compressed bytes
cr := &countingReader{Reader: reader}
cw := &countingWriter{Writer: writer}
// Compression is just above counting
zr := newLZ4Reader(cr)
zw := newLZ4Writer(cw)
// We buffer writes on top of compression.
// The LZ4 reader is already internally buffered
wb := bufio.NewWriterSize(zw, 65536)
c := rawConnection{
id: nodeID,
name: name,
receiver: nativeModel{receiver},
state: stateInitial,
cr: cr,
xr: xdr.NewReader(zr),
cw: cw,
wb: wb,
xw: xdr.NewWriter(wb),
awaiting: make([]chan asyncResult, 0x1000),
outbox: make(chan []encodable),
outbox: make(chan hdrMsg),
nextID: make(chan int),
closed: make(chan struct{}),
}
@@ -162,7 +164,7 @@ func (c *rawConnection) Index(repo string, idx []FileInfo) error {
default:
}
c.idxMut.Lock()
c.send(header{0, -1, messageTypeIndex}, IndexMessage{repo, idx})
c.send(-1, messageTypeIndex, IndexMessage{repo, idx})
c.idxMut.Unlock()
return nil
}
@@ -175,7 +177,7 @@ func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error {
default:
}
c.idxMut.Lock()
c.send(header{0, -1, messageTypeIndexUpdate}, IndexMessage{repo, idx})
c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx})
c.idxMut.Unlock()
return nil
}
@@ -197,8 +199,7 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
c.awaiting[id] = rc
c.awaitingMut.Unlock()
ok := c.send(header{0, id, messageTypeRequest},
RequestMessage{repo, name, uint64(offset), uint32(size)})
ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)})
if !ok {
return nil, ErrClosed
}
@@ -212,7 +213,7 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
// ClusterConfig send the cluster configuration message to the peer and returns any error
func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
c.send(header{0, -1, messageTypeClusterConfig}, config)
c.send(-1, messageTypeClusterConfig, config)
}
func (c *rawConnection) ping() bool {
@@ -228,7 +229,7 @@ func (c *rawConnection) ping() bool {
c.awaiting[id] = rc
c.awaitingMut.Unlock()
ok := c.send(header{0, id, messageTypePing})
ok := c.send(id, messageTypePing, nil)
if !ok {
return false
}
@@ -249,68 +250,53 @@ func (c *rawConnection) readerLoop() (err error) {
default:
}
var hdr header
hdr.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
hdr, msg, err := c.readMessage()
if err != nil {
return err
}
if hdr.version != 0 {
return fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
}
switch hdr.msgType {
case messageTypeIndex:
if c.state < stateCCRcvd {
return fmt.Errorf("protocol error: index message in state %d", c.state)
}
if err := c.handleIndex(); err != nil {
return err
}
c.handleIndex(msg.(IndexMessage))
c.state = stateIdxRcvd
case messageTypeIndexUpdate:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: index update message in state %d", c.state)
}
if err := c.handleIndexUpdate(); err != nil {
return err
}
c.handleIndexUpdate(msg.(IndexMessage))
case messageTypeRequest:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: request message in state %d", c.state)
}
if err := c.handleRequest(hdr); err != nil {
return err
}
// Requests are handled asynchronously
go c.handleRequest(hdr.msgID, msg.(RequestMessage))
case messageTypeResponse:
if c.state < stateIdxRcvd {
return fmt.Errorf("protocol error: response message in state %d", c.state)
}
if err := c.handleResponse(hdr); err != nil {
return err
}
c.handleResponse(hdr.msgID, msg.(ResponseMessage))
case messageTypePing:
c.send(header{0, hdr.msgID, messageTypePong})
c.send(hdr.msgID, messageTypePong, EmptyMessage{})
case messageTypePong:
c.handlePong(hdr)
c.handlePong(hdr.msgID)
case messageTypeClusterConfig:
if c.state != stateInitial {
return fmt.Errorf("protocol error: cluster config message in state %d", c.state)
}
if err := c.handleClusterConfig(); err != nil {
return err
}
go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage))
c.state = stateCCRcvd
case messageTypeClose:
if err := c.handleClose(); err != nil {
return err
}
return errors.New(msg.(CloseMessage).Reason)
default:
return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
@@ -318,114 +304,153 @@ func (c *rawConnection) readerLoop() (err error) {
}
}
func (c *rawConnection) handleIndex() error {
var im IndexMessage
im.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
return err
func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
if cap(c.rdbuf0) < 8 {
c.rdbuf0 = make([]byte, 8)
} else {
if debug {
l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
}
c.receiver.Index(c.id, im.Repository, im.Files)
c.rdbuf0 = c.rdbuf0[:8]
}
_, err = io.ReadFull(c.cr, c.rdbuf0)
if err != nil {
return
}
return nil
}
func (c *rawConnection) handleIndexUpdate() error {
var im IndexMessage
im.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
return err
hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4]))
msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8]))
if debug {
l.Debugf("read header %v (msglen=%d)", hdr, msglen)
}
if cap(c.rdbuf0) < msglen {
c.rdbuf0 = make([]byte, msglen)
} else {
if debug {
l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
c.rdbuf0 = c.rdbuf0[:msglen]
}
_, err = io.ReadFull(c.cr, c.rdbuf0)
if err != nil {
return
}
if debug {
l.Debugf("read %d bytes", len(c.rdbuf0))
}
msgBuf := c.rdbuf0
if hdr.compression {
c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)]
c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0)
if err != nil {
return
}
msgBuf = c.rdbuf1
if debug {
l.Debugf("decompressed to %d bytes", len(msgBuf))
}
c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
}
return nil
if debug {
if len(msgBuf) > 1024 {
l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024]))
} else {
l.Debugf("message data:\n%s", hex.Dump(msgBuf))
}
}
switch hdr.msgType {
case messageTypeIndex, messageTypeIndexUpdate:
var idx IndexMessage
err = idx.UnmarshalXDR(msgBuf)
msg = idx
case messageTypeRequest:
var req RequestMessage
err = req.UnmarshalXDR(msgBuf)
msg = req
case messageTypeResponse:
var resp ResponseMessage
err = resp.UnmarshalXDR(msgBuf)
msg = resp
case messageTypePing, messageTypePong:
msg = EmptyMessage{}
case messageTypeClusterConfig:
var cc ClusterConfigMessage
err = cc.UnmarshalXDR(msgBuf)
msg = cc
case messageTypeClose:
var cm CloseMessage
err = cm.UnmarshalXDR(msgBuf)
msg = cm
default:
err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
}
return
}
func (c *rawConnection) handleRequest(hdr header) error {
var req RequestMessage
req.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
return err
func (c *rawConnection) handleIndex(im IndexMessage) {
if debug {
l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
}
go c.processRequest(hdr.msgID, req)
return nil
c.receiver.Index(c.id, im.Repository, im.Files)
}
func (c *rawConnection) handleResponse(hdr header) error {
data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
if err := c.xr.Error(); err != nil {
return err
func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
if debug {
l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files))
}
c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
}
func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
c.send(msgID, messageTypeResponse, ResponseMessage{data})
}
func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
c.awaitingMut.Lock()
if rc := c.awaiting[hdr.msgID]; rc != nil {
c.awaiting[hdr.msgID] = nil
rc <- asyncResult{data, nil}
if rc := c.awaiting[msgID]; rc != nil {
c.awaiting[msgID] = nil
rc <- asyncResult{resp.Data, nil}
close(rc)
}
c.awaitingMut.Unlock()
return nil
}
func (c *rawConnection) handlePong(hdr header) {
func (c *rawConnection) handlePong(msgID int) {
c.awaitingMut.Lock()
if rc := c.awaiting[hdr.msgID]; rc != nil {
c.awaiting[hdr.msgID] = nil
if rc := c.awaiting[msgID]; rc != nil {
c.awaiting[msgID] = nil
rc <- asyncResult{}
close(rc)
}
c.awaitingMut.Unlock()
}
func (c *rawConnection) handleClusterConfig() error {
var cm ClusterConfigMessage
cm.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
return err
} else {
go c.receiver.ClusterConfig(c.id, cm)
}
return nil
}
func (c *rawConnection) handleClose() error {
var cm CloseMessage
cm.decodeXDR(c.xr)
if err := c.xr.Error(); err != nil {
return err
}
return errors.New(cm.Reason)
}
type encodable interface {
encodeXDR(*xdr.Writer) (int, error)
}
type encodableBytes []byte
func (e encodableBytes) encodeXDR(xw *xdr.Writer) (int, error) {
return xw.WriteBytes(e)
}
func (c *rawConnection) send(h header, es ...encodable) bool {
if h.msgID < 0 {
func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool {
if msgID < 0 {
select {
case id := <-c.nextID:
h.msgID = id
msgID = id
case <-c.closed:
return false
}
}
msg := append([]encodable{h}, es...)
hdr := header{
version: 0,
msgID: msgID,
msgType: msgType,
}
select {
case c.outbox <- msg:
case c.outbox <- hdrMsg{hdr, msg}:
return true
case <-c.closed:
return false
@@ -433,13 +458,71 @@ func (c *rawConnection) send(h header, es ...encodable) bool {
}
func (c *rawConnection) writerLoop() {
var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
var uncBuf []byte // buffer for uncompressed message, kept and reused
for {
var tempBuf []byte
var err error
select {
case es := <-c.outbox:
for _, e := range es {
e.encodeXDR(c.xw)
case hm := <-c.outbox:
if hm.msg != nil {
// Uncompressed message in uncBuf
uncBuf = hm.msg.AppendXDR(uncBuf[:0])
if len(uncBuf) >= MinCompressedSize {
// Use compression for large messages
hm.hdr.compression = true
// Make sure we have enough space for the compressed message plus header in msgBug
msgBuf = msgBuf[:cap(msgBuf)]
if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
msgBuf = make([]byte, maxLen)
}
// Compressed is written to msgBuf, we keep tb for the length only
tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
msgBuf = msgBuf[0 : len(tempBuf)+8]
if debug {
l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
}
} else {
// No point in compressing very short messages
hm.hdr.compression = false
msgBuf = msgBuf[:cap(msgBuf)]
if l := len(uncBuf) + 8; l > len(msgBuf) {
msgBuf = make([]byte, l)
}
binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
msgBuf = msgBuf[0 : len(uncBuf)+8]
copy(msgBuf[8:], uncBuf)
if debug {
l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
}
}
} else {
if debug {
l.Debugf("write empty message; %v", hm.hdr)
}
binary.BigEndian.PutUint32(msgBuf[4:8], 0)
msgBuf = msgBuf[:8]
}
if err := c.flush(); err != nil {
binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))
if err == nil {
var n int
n, err = c.cw.Write(msgBuf)
if debug {
l.Debugf("wrote %d bytes on the wire", n)
}
}
if err != nil {
c.close(err)
return
}
@@ -449,16 +532,6 @@ func (c *rawConnection) writerLoop() {
}
}
func (c *rawConnection) flush() error {
if err := c.xw.Error(); err != nil {
return err
}
if err := c.wb.Flush(); err != nil {
return err
}
return nil
}
func (c *rawConnection) close(err error) {
c.once.Do(func() {
close(c.closed)
@@ -494,13 +567,13 @@ func (c *rawConnection) pingerLoop() {
for {
select {
case <-ticker:
if d := time.Since(c.xr.LastRead()); d < pingIdleTime {
if d := time.Since(c.cr.Last()); d < pingIdleTime {
if debug {
l.Debugln(c.id, "ping skipped after rd", d)
}
continue
}
if d := time.Since(c.xw.LastWrite()); d < pingIdleTime {
if d := time.Since(c.cw.Last()); d < pingIdleTime {
if debug {
l.Debugln(c.id, "ping skipped after wr", d)
}
@@ -532,12 +605,6 @@ func (c *rawConnection) pingerLoop() {
}
}
func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
c.send(header{0, msgID, messageTypeResponse}, encodableBytes(data))
}
type Statistics struct {
At time.Time
InBytesTotal uint64