perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/protocol, lib/discover, lib/db: Use protocol buffer serialization (fixes #3080)

Browse Source 
This changes the BEP protocol to use protocol buffer serialization
instead of XDR, and therefore also the database format. The local
discovery protocol is also updated to be protocol buffer format.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3276
LGTM: AudriusButkevicius
This commit is contained in:
Jakob Borg
2016-07-04 10:40:29 +00:00
committed by Audrius Butkevicius
parent 21f5b16e47
commit fa0101bd60
269 changed files with 477296 additions and 4175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
cmd/stdisco/main.go
View File

@@ -9,6 +9,7 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/binary"
"flag"
"log"
"strings"
@@ -66,24 +67,25 @@ func recv(bc beacon.Interface) {
seen := make(map[string]bool)
for {
data, src := bc.Recv()
var ann discover.Announce
ann.UnmarshalXDR(data)
if m := binary.BigEndian.Uint32(data); m != discover.Magic {
log.Printf("Incorrect magic %x in announcement from %v", m, src)
continue
}
if bytes.Equal(ann.This.ID, myID) {
var ann discover.Announce
ann.Unmarshal(data[4:])
if bytes.Equal(ann.ID, myID) {
// This is one of our own fake packets, don't print it.
continue
}
// Print announcement details for the first packet from a given
// device ID and source address, or if -all was given.
key := string(ann.This.ID) + src.String()
key := string(ann.ID) + src.String()
if all || !seen[key] {
log.Printf("Announcement from %v\n", src)
log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
for _, dev := range ann.Extra {
log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
}
log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.ID), strings.Join(ann.Addresses, ", "))
seen[key] = true
}
}
@@ -92,15 +94,10 @@ func recv(bc beacon.Interface) {
// sends fake discovery announcements once every second
func send(bc beacon.Interface) {
ann := discover.Announce{
Magic: discover.AnnouncementMagic,
This: discover.Device{
ID: myID,
Addresses: []discover.Address{
{URL: "tcp://fake.example.com:12345"},
},
},
ID: myID,
Addresses: []string{"tcp://fake.example.com:12345"},
}
bs, _ := ann.MarshalXDR()
bs, _ := ann.Marshal()
for {
bc.Send(bs)
@@ -108,15 +105,6 @@ func send(bc beacon.Interface) {
}
}
// returns the list of address URLs
func addrStrs(dev discover.Device) []string {
ss := make([]string, len(dev.Addresses))
for i, addr := range dev.Addresses {
ss[i] = addr.URL
}
return ss
}
// returns a random but recognizable device ID
func randomDeviceID() []byte {
var id [32]byte

53
cmd/stindex/dump.go
View File

@@ -13,50 +13,61 @@ import (
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
)
func dump(ldb *leveldb.DB) {
func dump(ldb *db.Instance) {
it := ldb.NewIterator(nil, nil)
var dev protocol.DeviceID
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := nulString(key[1 : 1+64])
devBytes := key[1+64 : 1+64+32]
name := nulString(key[1+64+32:])
copy(dev[:], devBytes)
fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
var f protocol.FileInfo
err := f.UnmarshalXDR(it.Value())
err := f.Unmarshal(it.Value())
if err != nil {
log.Fatal(err)
}
fmt.Printf(" N:%q\n F:%#o\n M:%d\n V:%v\n S:%d\n B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
fmt.Printf(" V:%v\n", f)
case db.KeyTypeGlobal:
folder := nulString(key[1 : 1+64])
name := nulString(key[1+64:])
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
var flv db.VersionList
flv.UnmarshalXDR(it.Value())
fmt.Printf("[global] F:%q N:%q V: %s\n", folder, name, flv)
flv.Unmarshal(it.Value())
fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
case db.KeyTypeBlock:
folder := nulString(key[1 : 1+64])
hash := key[1+64 : 1+64+32]
name := nulString(key[1+64+32:])
fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
case db.KeyTypeDeviceStatistic:
fmt.Printf("[dstat]\n %x\n %x\n", it.Key(), it.Value())
fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeFolderStatistic:
fmt.Printf("[fstat]\n %x\n %x\n", it.Key(), it.Value())
fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeVirtualMtime:
fmt.Printf("[mtime]\n %x\n %x\n", it.Key(), it.Value())
fmt.Printf("[mtime] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeFolderIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
case db.KeyTypeDeviceIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
val := it.Value()
if len(val) == 0 {
fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
} else {
dev := protocol.DeviceIDFromBytes(val)
fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
}
default:
fmt.Printf("[???]\n %x\n %x\n", it.Key(), it.Value())

37
cmd/stindex/dumpsize.go
View File

@@ -8,11 +8,10 @@ package main
import (
"container/heap"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
)
type SizedElement struct {
@@ -38,33 +37,31 @@ func (h *ElementHeap) Pop() interface{} {
return x
}
func dumpsize(ldb *leveldb.DB) {
func dumpsize(ldb *db.Instance) {
h := &ElementHeap{}
heap.Init(h)
it := ldb.NewIterator(nil, nil)
var dev protocol.DeviceID
var ele SizedElement
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := nulString(key[1 : 1+64])
devBytes := key[1+64 : 1+64+32]
name := nulString(key[1+64+32:])
copy(dev[:], devBytes)
ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
case db.KeyTypeGlobal:
folder := nulString(key[1 : 1+64])
name := nulString(key[1+64:])
ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
case db.KeyTypeBlock:
folder := nulString(key[1 : 1+64])
hash := key[1+64 : 1+64+32]
name := nulString(key[1+64+32:])
ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
case db.KeyTypeDeviceStatistic:
ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
@@ -75,6 +72,14 @@ func dumpsize(ldb *leveldb.DB) {
case db.KeyTypeVirtualMtime:
ele.key = fmt.Sprintf("MTIME:%s", key[1:])
case db.KeyTypeFolderIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
case db.KeyTypeDeviceIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
default:
ele.key = fmt.Sprintf("UNKNOWN:%x", key)
}

11
cmd/stindex/main.go
View File

@@ -13,8 +13,7 @@ import (
"os"
"path/filepath"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syncthing/syncthing/lib/db"
)
func main() {
@@ -28,16 +27,12 @@ func main() {
path := flag.Arg(0)
if path == "" {
path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
}
fmt.Println("Path:", path)
ldb, err := leveldb.OpenFile(path, &opt.Options{
ErrorIfMissing: true,
Strict: opt.StrictAll,
OpenFilesCacheCapacity: 100,
})
ldb, err := db.Open(path)
if err != nil {
log.Fatal(err)
}

37
cmd/syncthing/gui.go
View File

@@ -1173,13 +1173,17 @@ type jsonFileInfo protocol.FileInfo
func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
return json.Marshal(map[string]interface{}{
"name": f.Name,
"size": protocol.FileInfo(f).Size(),
"flags": fmt.Sprintf("%#o", f.Flags),
"modified": time.Unix(f.Modified, 0),
"localVersion": f.LocalVersion,
"numBlocks": len(f.Blocks),
"version": jsonVersionVector(f.Version),
"name": f.Name,
"type": f.Type,
"size": f.Size,
"permissions": fmt.Sprintf("%#o", f.Permissions),
"deleted": f.Deleted,
"invalid": f.Invalid,
"noPermissions": f.NoPermissions,
"modified": time.Unix(f.Modified, 0),
"localVersion": f.LocalVersion,
"numBlocks": len(f.Blocks),
"version": jsonVersionVector(f.Version),
})
}
@@ -1187,20 +1191,23 @@ type jsonDBFileInfo db.FileInfoTruncated
func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
return json.Marshal(map[string]interface{}{
"name": f.Name,
"size": db.FileInfoTruncated(f).Size(),
"flags": fmt.Sprintf("%#o", f.Flags),
"modified": time.Unix(f.Modified, 0),
"localVersion": f.LocalVersion,
"version": jsonVersionVector(f.Version),
"name": f.Name,
"type": f.Type,
"size": f.Size,
"permissions": fmt.Sprintf("%#o", f.Permissions),
"deleted": f.Deleted,
"invalid": f.Invalid,
"noPermissions": f.NoPermissions,
"modified": time.Unix(f.Modified, 0),
"localVersion": f.LocalVersion,
})
}
type jsonVersionVector protocol.Vector
func (v jsonVersionVector) MarshalJSON() ([]byte, error) {
res := make([]string, len(v))
for i, c := range v {
res := make([]string, len(v.Counters))
for i, c := range v.Counters {
res[i] = fmt.Sprintf("%v:%d", c.ID, c.Value)
}
return json.Marshal(res)

2
cmd/syncthing/locations.go
View File

@@ -48,7 +48,7 @@ var locations = map[locationEnum]string{
locKeyFile: "${config}/key.pem",
locHTTPSCertFile: "${config}/https-cert.pem",
locHTTPSKeyFile: "${config}/https-key.pem",
locDatabase: "${config}/index-v0.13.0.db",
locDatabase: "${config}/index-v0.14.0.db",
locLogFile: "${config}/syncthing.log", // -logfile on Windows
locCsrfTokens: "${config}/csrftokens.txt",
locPanicLog: "${config}/panic-${timestamp}.log",

2
cmd/syncthing/main.go
View File

@@ -690,7 +690,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
if device == myID {
continue
}
m.Index(device, folderCfg.ID, nil, 0, nil)
m.Index(device, folderCfg.ID, nil)
}
m.StartFolder(folderCfg.ID)
}