perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/protocol, lib/discover, lib/db: Use protocol buffer serialization (fixes #3080)

Browse Source 
This changes the BEP protocol to use protocol buffer serialization
instead of XDR, and therefore also the database format. The local
discovery protocol is also updated to be protocol buffer format.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3276
LGTM: AudriusButkevicius
This commit is contained in:
Jakob Borg
2016-07-04 10:40:29 +00:00
committed by Audrius Butkevicius
parent 21f5b16e47
commit fa0101bd60
269 changed files with 477296 additions and 4175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/connections/service.go
View File

@@ -162,8 +162,17 @@ next:
if err != nil {
if protocol.IsVersionMismatch(err) {
// The error will be a relatively user friendly description
// of what's wrong with the version compatibility
msg := fmt.Sprintf("Connecting to %s (%s): %s", remoteID, c.RemoteAddr(), err)
// of what's wrong with the version compatibility. By
// default identify the other side by device ID and IP.
remote := fmt.Sprintf("%v (%v)", remoteID, c.RemoteAddr())
if hello.DeviceName != "" {
// If the name was set in the hello return, use that to
// give the user more info about which device is the
// affected one. It probably says more than the remote
// IP.
remote = fmt.Sprintf("%q (%s %s, %v)", hello.DeviceName, hello.ClientName, hello.ClientVersion, remoteID)
}
msg := fmt.Sprintf("Connecting to %s: %s", remote, err)
warningFor(remoteID, msg)
} else {
// It's something else - connection reset or whatever

2
lib/connections/structs.go
View File

@@ -70,7 +70,7 @@ type Model interface {
ConnectedTo(remoteID protocol.DeviceID) bool
IsPaused(remoteID protocol.DeviceID) bool
OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult)
GetHello(protocol.DeviceID) protocol.Version13HelloMessage
GetHello(protocol.DeviceID) protocol.HelloIntf
}
// serviceFunc wraps a function to create a suture.Service without stop

26
lib/db/blockmap_test.go
View File

@@ -73,7 +73,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
f3.Flags |= protocol.FlagDirectory
f3.Type = protocol.FileInfoTypeDirectory
err := m.Add([]protocol.FileInfo{f1, f2, f3})
if err != nil {
@@ -99,9 +99,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
return true
})
f3.Flags = f1.Flags
f1.Flags |= protocol.FlagDeleted
f2.Flags |= protocol.FlagInvalid
f3.Permissions = f1.Permissions
f3.Deleted = f1.Deleted
f3.Invalid = f1.Invalid
f1.Deleted = true
f2.Invalid = true
// Should remove
err = m.Update([]protocol.FileInfo{f1, f2, f3})
@@ -145,9 +147,15 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
t.Fatal("db not empty")
}
f1.Flags = 0
f2.Flags = 0
f3.Flags = 0
f1.Deleted = false
f1.Invalid = false
f1.Permissions = 0
f2.Deleted = false
f2.Invalid = false
f2.Permissions = 0
f3.Deleted = false
f3.Invalid = false
f3.Permissions = 0
}
func TestBlockFinderLookup(t *testing.T) {
@@ -187,7 +195,7 @@ func TestBlockFinderLookup(t *testing.T) {
t.Fatal("Incorrect count", counter)
}
f1.Flags |= protocol.FlagDeleted
f1.Deleted = true
err = m1.Update([]protocol.FileInfo{f1})
if err != nil {
@@ -212,7 +220,7 @@ func TestBlockFinderLookup(t *testing.T) {
t.Fatal("Incorrect count")
}
f1.Flags = 0
f1.Deleted = false
}
func TestBlockFinderFix(t *testing.T) {

20
lib/db/leveldb.go
View File

@@ -4,9 +4,6 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o leveldb_xdr.go leveldb.go
package db
import (
@@ -46,25 +43,16 @@ const (
KeyTypeDeviceIdx
)
type fileVersion struct {
version protocol.Vector
device []byte
}
type VersionList struct {
versions []fileVersion
}
func (l VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range l.versions {
for i, v := range l.Versions {
if i > 0 {
b.WriteString(", ")
}
copy(id[:], v.device)
fmt.Fprintf(&b, "{%d, %v}", v.version, id)
copy(id[:], v.Device)
fmt.Fprintf(&b, "{%d, %v}", v.Version, id)
}
b.WriteString("}")
return b.String()
@@ -101,7 +89,7 @@ func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) {
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
err = f.Unmarshal(bs)
if err != nil {
panic(err)
}

114
lib/db/leveldb_convert.go
View File

@@ -1,114 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"github.com/syndtr/goleveldb/leveldb"
)
// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
// avoid having to do rescan. The change is in the key format for folder
// labels, so we basically just iterate over the database rewriting keys as
// necessary.
func convertKeyFormat(from, to *leveldb.DB) error {
l.Infoln("Converting database key format")
blocks, files, globals, unchanged := 0, 0, 0, 0
dbi := newDBInstance(to)
i := from.NewIterator(nil, nil)
for i.Next() {
key := i.Key()
switch key[0] {
case KeyTypeBlock:
folder, file := oldFromBlockKey(key)
folderIdx := dbi.folderIdx.ID([]byte(folder))
hash := key[1+64:]
newKey := blockKeyInto(nil, hash, folderIdx, file)
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
blocks++
case KeyTypeDevice:
newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
files++
case KeyTypeGlobal:
newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
globals++
case KeyTypeVirtualMtime:
// Cannot be converted, we drop it instead :(
default:
if err := to.Put(key, i.Value(), nil); err != nil {
return err
}
unchanged++
}
}
l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)
return nil
}
func oldDeviceKeyFolder(key []byte) []byte {
folder := key[1 : 1+64]
izero := bytes.IndexByte(folder, 0)
if izero < 0 {
return folder
}
return folder[:izero]
}
func oldDeviceKeyDevice(key []byte) []byte {
return key[1+64 : 1+64+32]
}
func oldDeviceKeyName(key []byte) []byte {
return key[1+64+32:]
}
func oldGlobalKeyName(key []byte) []byte {
return key[1+64:]
}
func oldGlobalKeyFolder(key []byte) []byte {
folder := key[1 : 1+64]
izero := bytes.IndexByte(folder, 0)
if izero < 0 {
return folder
}
return folder[:izero]
}
func oldFromBlockKey(data []byte) (string, string) {
if len(data) < 1+64+32+1 {
panic("Incorrect key length")
}
if data[0] != KeyTypeBlock {
panic("Incorrect key type")
}
file := string(data[1+64+32:])
slice := data[1 : 1+64]
izero := bytes.IndexByte(slice, 0)
if izero > -1 {
return string(slice[:izero]), file
}
return string(slice), file
}

136
lib/db/leveldb_convert_test.go
View File

@@ -1,136 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"archive/zip"
"io"
"os"
"path/filepath"
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
func TestLabelConversion(t *testing.T) {
os.RemoveAll("testdata/oldformat.db")
defer os.RemoveAll("testdata/oldformat.db")
os.RemoveAll("testdata/newformat.db")
defer os.RemoveAll("testdata/newformat.db")
if err := unzip("testdata/oldformat.db.zip", "testdata"); err != nil {
t.Fatal(err)
}
odb, err := leveldb.OpenFile("testdata/oldformat.db", nil)
if err != nil {
t.Fatal(err)
}
ldb, err := leveldb.OpenFile("testdata/newformat.db", nil)
if err != nil {
t.Fatal(err)
}
if err = convertKeyFormat(odb, ldb); err != nil {
t.Fatal(err)
}
ldb.Close()
odb.Close()
inst, err := Open("testdata/newformat.db")
if err != nil {
t.Fatal(err)
}
fs := NewFileSet("default", inst)
files, deleted, _ := fs.GlobalSize()
if files+deleted != 953 {
// Expected number of global entries determined by
// ../../bin/stindex testdata/oldformat.db/ | grep global | grep -c default
t.Errorf("Conversion error, global list differs (%d != 953)", files+deleted)
}
files, deleted, _ = fs.LocalSize()
if files+deleted != 953 {
t.Errorf("Conversion error, device list differs (%d != 953)", files+deleted)
}
f := NewBlockFinder(inst)
// [block] F:"default" H:1c25dea9003cc16216e2a22900be1ec1cc5aaf270442904e2f9812c314e929d8 N:"f/f2/f25f1b3e6e029231b933531b2138796d" I:3
h := []byte{0x1c, 0x25, 0xde, 0xa9, 0x00, 0x3c, 0xc1, 0x62, 0x16, 0xe2, 0xa2, 0x29, 0x00, 0xbe, 0x1e, 0xc1, 0xcc, 0x5a, 0xaf, 0x27, 0x04, 0x42, 0x90, 0x4e, 0x2f, 0x98, 0x12, 0xc3, 0x14, 0xe9, 0x29, 0xd8}
found := 0
f.Iterate([]string{"default"}, h, func(folder, file string, idx int32) bool {
if folder == "default" && file == filepath.FromSlash("f/f2/f25f1b3e6e029231b933531b2138796d") && idx == 3 {
found++
}
return true
})
if found != 1 {
t.Errorf("Found %d blocks instead of expected 1", found)
}
inst.Close()
}
func unzip(src, dest string) error {
r, err := zip.OpenReader(src)
if err != nil {
return err
}
defer func() {
if err := r.Close(); err != nil {
panic(err)
}
}()
os.MkdirAll(dest, 0755)
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
panic(err)
}
}()
path := filepath.Join(dest, f.Name)
if f.FileInfo().IsDir() {
os.MkdirAll(path, f.Mode())
} else {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
panic(err)
}
}()
_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}
for _, f := range r.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}
return nil
}

135
lib/db/leveldb_dbinstance.go
View File

@@ -10,12 +10,10 @@ import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb"
@@ -48,16 +46,6 @@ func Open(file string) (*Instance, error) {
WriteBuffer: 4 << 20,
}
if _, err := os.Stat(file); os.IsNotExist(err) {
// The file we are looking to open does not exist. This may be the
// first launch so we should look for an old version and try to
// convert it.
if err := checkConvertDatabase(file); err != nil {
l.Infoln("Converting old database:", err)
l.Infoln("Will rescan from scratch.")
}
}
db, err := leveldb.OpenFile(file, opts)
if leveldbIsCorrupted(err) {
db, err = leveldb.RecoverFile(file, opts)
@@ -151,12 +139,12 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
case moreFs && moreDb && cmp == 0:
// File exists on both sides - compare versions. We might get an
// update with the same version and different flags if a device has
// marked a file as invalid, so handle that too.
// update with the same version if a device has marked a file as
// invalid, so handle that too.
l.Debugln("generic replace; exists - compare")
var ef FileInfoTruncated
ef.UnmarshalXDR(dbi.Value())
if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
ef.Unmarshal(dbi.Value())
if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Invalid != ef.Invalid {
l.Debugln("generic replace; differs - insert")
if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
maxLocalVer = lv
@@ -232,13 +220,12 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
}
var ef FileInfoTruncated
err = ef.UnmarshalXDR(bs)
err = ef.Unmarshal(bs)
if err != nil {
panic(err)
}
// Flags might change without the version being bumped when we set the
// invalid flag on an existing file.
if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
// The Invalid flag might change without the version being bumped.
if !ef.Version.Equal(f.Version) || ef.Invalid != f.Invalid {
if isLocalDevice {
localSize.removeFile(ef)
localSize.addFile(f)
@@ -308,7 +295,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
// struct, which in turn references the buffer it was unmarshalled
// from. dbi.Value() just returns an internal slice that it reuses, so
// we need to copy it.
err := f.UnmarshalXDR(append([]byte{}, dbi.Value()...))
err := f.Unmarshal(append([]byte{}, dbi.Value()...))
if err != nil {
panic(err)
}
@@ -347,16 +334,16 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
}
var vl VersionList
err = vl.UnmarshalXDR(bs)
err = vl.Unmarshal(bs)
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(k)
panic("no versions?")
}
k = db.deviceKey(folder, vl.versions[0].device, file)
k = db.deviceKey(folder, vl.Versions[0].Device, file)
bs, err = t.Get(k, nil)
if err != nil {
panic(err)
@@ -384,11 +371,11 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
var fk []byte
for dbi.Next() {
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
@@ -398,13 +385,13 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
return
}
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[0].Device, name)
bs, err := t.Get(fk, nil)
if err != nil {
l.Debugf("folder: %q (%x)", folder, folder)
l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
l.Debugf("vl: %v", vl)
l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
l.Debugf("vl.Versions[0].Device: %x", vl.Versions[0].Device)
l.Debugf("name: %q (%x)", name, name)
l.Debugf("fk: %q", fk)
l.Debugf("fk: %x %x %x",
@@ -436,17 +423,17 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
}
var vl VersionList
err = vl.UnmarshalXDR(bs)
err = vl.Unmarshal(bs)
if err != nil {
panic(err)
}
var devices []protocol.DeviceID
for _, v := range vl.versions {
if !v.version.Equal(vl.versions[0].version) {
for _, v := range vl.Versions {
if !v.Version.Equal(vl.Versions[0].Version) {
break
}
n := protocol.DeviceIDFromBytes(v.device)
n := protocol.DeviceIDFromBytes(v.Device)
devices = append(devices, n)
}
@@ -464,11 +451,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
nextFile:
for dbi.Next() {
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
@@ -476,29 +463,29 @@ nextFile:
have := false // If we have the file, any version
need := false // If we have a lower version of the file
var haveVersion protocol.Vector
for _, v := range vl.versions {
if bytes.Equal(v.device, device) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
have = true
haveVersion = v.version
haveVersion = v.Version
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
need = !v.version.GreaterEqual(vl.versions[0].version)
need = !v.Version.GreaterEqual(vl.Versions[0].Version)
break
}
}
if need || !have {
name := db.globalKeyName(dbi.Key())
needVersion := vl.versions[0].version
needVersion := vl.Versions[0].Version
nextVersion:
for i := range vl.versions {
if !vl.versions[i].version.Equal(needVersion) {
for i := range vl.Versions {
if !vl.Versions[i].Version.Equal(needVersion) {
// We haven't found a valid copy of the file with the needed version.
continue nextFile
}
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[i].Device, name)
bs, err := t.Get(fk, nil)
if err != nil {
var id protocol.DeviceID
@@ -528,7 +515,7 @@ nextFile:
continue nextFile
}
l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.Versions[0].Version)
if cont := fn(gf); !cont {
return
@@ -601,7 +588,7 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
for dbi.Next() {
gk := dbi.Key()
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
@@ -613,8 +600,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
name := db.globalKeyName(gk)
var newVL VersionList
for i, version := range vl.versions {
fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
for i, version := range vl.Versions {
fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.Device, name)
_, err := t.Get(fk, nil)
if err == leveldb.ErrNotFound {
@@ -623,10 +610,10 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
if err != nil {
panic(err)
}
newVL.versions = append(newVL.versions, version)
newVL.Versions = append(newVL.Versions, version)
if i == 0 {
fi, ok := t.getFile(folder, version.device, name)
fi, ok := t.getFile(folder, version.Device, name)
if !ok {
panic("nonexistent global master file")
}
@@ -634,8 +621,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
}
}
if len(newVL.versions) != len(vl.versions) {
t.Put(dbi.Key(), newVL.MustMarshalXDR())
if len(newVL.Versions) != len(vl.Versions) {
t.Put(dbi.Key(), mustMarshal(&newVL))
t.checkFlush()
}
}
@@ -715,12 +702,12 @@ func (db *Instance) globalKeyFolder(key []byte) []byte {
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf FileInfoTruncated
err := tf.UnmarshalXDR(bs)
err := tf.Unmarshal(bs)
return tf, err
}
var tf protocol.FileInfo
err := tf.UnmarshalXDR(bs)
err := tf.Unmarshal(bs)
return tf, err
}
@@ -740,50 +727,6 @@ func leveldbIsCorrupted(err error) bool {
return false
}
// checkConvertDatabase tries to convert an existing old (v0.11) database to
// new (v0.13) format.
func checkConvertDatabase(dbFile string) error {
oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db")
if _, err := os.Stat(oldLoc); os.IsNotExist(err) {
// The old database file does not exist; that's ok, continue as if
// everything succeeded.
return nil
} else if err != nil {
// Any other error is weird.
return err
}
// There exists a database in the old format. We run a one time
// conversion from old to new.
fromDb, err := leveldb.OpenFile(oldLoc, nil)
if err != nil {
return err
}
toDb, err := leveldb.OpenFile(dbFile, nil)
if err != nil {
return err
}
err = convertKeyFormat(fromDb, toDb)
if err != nil {
return err
}
err = toDb.Close()
if err != nil {
return err
}
// We've done this one, we don't want to do it again (if the user runs
// -reset or so). We don't care too much about errors any more at this stage.
fromDb.Close()
osutil.Rename(oldLoc, oldLoc+".converted")
return nil
}
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
// fast lookups in both directions and persists to the database. Don't use for
// storing more items than fit comfortably in RAM.

70
lib/db/leveldb_transactions.go
View File

@@ -83,7 +83,7 @@ func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.Fi
name := []byte(file.Name)
nk := t.db.deviceKey(folder, device, name)
t.Put(nk, file.MustMarshalXDR())
t.Put(nk, mustMarshal(&file))
return file.LocalVersion
}
@@ -105,14 +105,14 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
var hasOldFile bool
// Remove the device from the current version list
if len(svl) != 0 {
err = fl.UnmarshalXDR(svl)
err = fl.Unmarshal(svl)
if err != nil {
panic(err)
}
for i := range fl.versions {
if bytes.Equal(fl.versions[i].device, device) {
if fl.versions[i].version.Equal(file.Version) {
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if fl.Versions[i].Version.Equal(file.Version) {
// No need to do anything
return false
}
@@ -120,29 +120,29 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
if i == 0 {
// Keep the current newest file around so we can subtract it from
// the globalSize if we replace it.
oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name)
oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
}
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
break
}
}
}
nv := fileVersion{
device: device,
version: file.Version,
nv := FileVersion{
Device: device,
Version: file.Version,
}
insertedAt := -1
// Find a position in the list to insert this file. The file at the front
// of the list is the newer, the "global".
for i := range fl.versions {
switch fl.versions[i].version.Compare(file.Version) {
for i := range fl.Versions {
switch fl.Versions[i].Version.Compare(file.Version) {
case protocol.Equal, protocol.Lesser:
// The version at this point in the list is equal to or lesser
// ("older") than us. We insert ourselves in front of it.
fl.versions = insertVersion(fl.versions, i, nv)
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
goto done
@@ -153,12 +153,12 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
// "Greater" in the condition above is just based on the device
// IDs in the version vector, which is not the only thing we use
// to determine the winner.)
of, ok := t.getFile(folder, fl.versions[i].device, name)
of, ok := t.getFile(folder, fl.Versions[i].Device, name)
if !ok {
panic("file referenced in version list does not exist")
}
if file.WinsConflict(of) {
fl.versions = insertVersion(fl.versions, i, nv)
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
goto done
}
@@ -166,8 +166,8 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
}
// We didn't find a position for an insert above, so append to the end.
fl.versions = append(fl.versions, nv)
insertedAt = len(fl.versions) - 1
fl.Versions = append(fl.Versions, nv)
insertedAt = len(fl.Versions) - 1
done:
if insertedAt == 0 {
@@ -178,9 +178,9 @@ done:
if hasOldFile {
// We have the old file that was removed at the head of the list.
globalSize.removeFile(oldFile)
} else if len(fl.versions) > 1 {
} else if len(fl.Versions) > 1 {
// The previous newest version is now at index 1, grab it from there.
oldFile, ok := t.getFile(folder, fl.versions[1].device, name)
oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name)
if !ok {
panic("file referenced in version list does not exist")
}
@@ -190,7 +190,7 @@ done:
}
l.Debugf("new global after update: %v", fl)
t.Put(gk, fl.MustMarshalXDR())
t.Put(gk, mustMarshal(&fl))
return true
}
@@ -210,14 +210,14 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
}
var fl VersionList
err = fl.UnmarshalXDR(svl)
err = fl.Unmarshal(svl)
if err != nil {
panic(err)
}
removed := false
for i := range fl.versions {
if bytes.Equal(fl.versions[i].device, device) {
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if i == 0 && globalSize != nil {
f, ok := t.getFile(folder, device, file)
if !ok {
@@ -226,18 +226,18 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
globalSize.removeFile(f)
removed = true
}
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
break
}
}
if len(fl.versions) == 0 {
if len(fl.Versions) == 0 {
t.Delete(gk)
} else {
l.Debugf("new global after remove: %v", fl)
t.Put(gk, fl.MustMarshalXDR())
t.Put(gk, mustMarshal(&fl))
if removed {
f, ok := t.getFile(folder, fl.versions[0].device, file)
f, ok := t.getFile(folder, fl.Versions[0].Device, file)
if !ok {
panic("new global is nonexistent file")
}
@@ -246,9 +246,21 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
}
}
func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
t := append(vl, fileVersion{})
func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
t := append(vl, FileVersion{})
copy(t[i+1:], t[i:])
t[i] = v
return t
}
type marshaller interface {
Marshal() ([]byte, error)
}
func mustMarshal(f marshaller) []byte {
bs, err := f.Marshal()
if err != nil {
panic(err)
}
return bs
}

142
lib/db/leveldb_xdr.go
View File

@@ -1,142 +0,0 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package db
import (
"github.com/calmh/xdr"
)
/*
fileVersion Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Vector Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ device (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct fileVersion {
Vector version;
opaque device<>;
}
*/
func (o fileVersion) XDRSize() int {
return o.version.XDRSize() +
4 + len(o.device) + xdr.Padding(len(o.device))
}
func (o fileVersion) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o fileVersion) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o fileVersion) MarshalXDRInto(m *xdr.Marshaller) error {
if err := o.version.MarshalXDRInto(m); err != nil {
return err
}
m.MarshalBytes(o.device)
return m.Error
}
func (o *fileVersion) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *fileVersion) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
(&o.version).UnmarshalXDRFrom(u)
o.device = u.UnmarshalBytes()
return u.Error
}
/*
VersionList Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of versions |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more fileVersion Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct VersionList {
fileVersion versions<>;
}
*/
func (o VersionList) XDRSize() int {
return 4 + xdr.SizeOfSlice(o.versions)
}
func (o VersionList) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o VersionList) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o VersionList) MarshalXDRInto(m *xdr.Marshaller) error {
m.MarshalUint32(uint32(len(o.versions)))
for i := range o.versions {
if err := o.versions[i].MarshalXDRInto(m); err != nil {
return err
}
}
return m.Error
}
func (o *VersionList) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *VersionList) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
_versionsSize := int(u.UnmarshalUint32())
if _versionsSize < 0 {
return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
} else if _versionsSize == 0 {
o.versions = nil
} else {
if _versionsSize <= len(o.versions) {
o.versions = o.versions[:_versionsSize]
} else {
o.versions = make([]fileVersion, _versionsSize)
}
for i := range o.versions {
(&o.versions[i]).UnmarshalXDRFrom(u)
}
}
return u.Error
}

11
lib/db/set.go
View File

@@ -31,9 +31,10 @@ type FileSet struct {
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
// FileInfoTruncated.
type FileIntf interface {
Size() int64
FileSize() int64
FileName() string
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
@@ -42,7 +43,7 @@ type FileIntf interface {
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
@@ -64,7 +65,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
} else {
s.files++
}
s.bytes += f.Size()
s.bytes += f.FileSize()
s.mut.Unlock()
}
@@ -79,7 +80,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
} else {
s.files--
}
s.bytes -= f.Size()
s.bytes -= f.FileSize()
if s.deleted < 0 || s.files < 0 {
panic("bug: removed more than added")
}

178
lib/db/set_test.go
View File

@@ -88,7 +88,7 @@ func (l fileList) String() string {
var b bytes.Buffer
b.WriteString("[]protocol.FileList{\n")
for _, f := range l {
fmt.Fprintf(&b, " %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
fmt.Fprintf(&b, " %q: #%d, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
}
b.WriteString("}")
return b.String()
@@ -100,35 +100,35 @@ func TestGlobalSet(t *testing.T) {
m := db.NewFileSet("test", ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
}
local1 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
}
localTot := fileList{
local0[0],
local0[1],
local0[2],
local0[3],
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
}
remote0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
}
remote1 := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
}
remoteTot := fileList{
remote0[0],
@@ -178,7 +178,7 @@ func TestGlobalSet(t *testing.T) {
} else {
globalFiles++
}
globalBytes += f.Size()
globalBytes += f.FileSize()
}
gsFiles, gsDeleted, gsBytes := m.GlobalSize()
if gsFiles != globalFiles {
@@ -208,7 +208,7 @@ func TestGlobalSet(t *testing.T) {
} else {
haveFiles++
}
haveBytes += f.Size()
haveBytes += f.FileSize()
}
lsFiles, lsDeleted, lsBytes := m.LocalSize()
if lsFiles != haveFiles {
@@ -303,23 +303,23 @@ func TestNeedWithInvalid(t *testing.T) {
s := db.NewFileSet("test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
}
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
remote1Have := fileList{
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
expectedNeed := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
s.Replace(protocol.LocalDeviceID, localHave)
@@ -340,10 +340,10 @@ func TestUpdateToInvalid(t *testing.T) {
s := db.NewFileSet("test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
s.Replace(protocol.LocalDeviceID, localHave)
@@ -355,7 +355,7 @@ func TestUpdateToInvalid(t *testing.T) {
t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
}
localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Invalid: true}
s.Update(protocol.LocalDeviceID, localHave[1:2])
have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -372,16 +372,16 @@ func TestInvalidAvailability(t *testing.T) {
s := db.NewFileSet("test", ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
remote1Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
s.Replace(remoteDevice0, remote0Have)
@@ -410,17 +410,17 @@ func TestGlobalReset(t *testing.T) {
m := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local)
@@ -448,23 +448,23 @@ func TestNeed(t *testing.T) {
m := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
shouldNeed := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local)
@@ -486,18 +486,18 @@ func TestLocalVersion(t *testing.T) {
m := db.NewFileSet("test", ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
local2 := []protocol.FileInfo{
local1[0],
// [1] deleted
local1[2],
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local1)
@@ -515,17 +515,17 @@ func TestListDropFolder(t *testing.T) {
s0 := db.NewFileSet("test0", ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
s0.Replace(protocol.LocalDeviceID, local1)
s1 := db.NewFileSet("test1", ldb)
local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
}
s1.Replace(remoteDevice0, local2)
@@ -566,24 +566,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
s := db.NewFileSet("test1", ldb)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
}
s.Replace(remoteDevice0, rem0)
rem1 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
}
s.Replace(remoteDevice1, rem1)
total := fileList{
// There's a valid copy of each file, so it should be merged
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
}
need := fileList(needList(s, protocol.LocalDeviceID))
@@ -609,7 +609,7 @@ func TestLongPath(t *testing.T) {
name := b.String() // 5000 characters
local := []protocol.FileInfo{
{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
s.Replace(protocol.LocalDeviceID, local)
@@ -633,7 +633,7 @@ func TestCommitted(t *testing.T) {
s := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
// Adding a file should increase the counter
@@ -659,12 +659,12 @@ func TestCommitted(t *testing.T) {
func BenchmarkUpdateOneFile(b *testing.B) {
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
// A longer name is more realistic and causes more allocations
protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
}
ldb, err := db.Open("testdata/benchmarkupdate.db")

57
lib/db/structs.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate go run ../../script/protofmt.go structs.proto
//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. structs.proto
package db
import (
"fmt"
"github.com/syncthing/syncthing/lib/protocol"
)
func (f FileInfoTruncated) String() string {
return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
}
func (f FileInfoTruncated) IsDeleted() bool {
return f.Deleted
}
func (f FileInfoTruncated) IsInvalid() bool {
return f.Invalid
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Type == protocol.FileInfoTypeDirectory
}
func (f FileInfoTruncated) IsSymlink() bool {
switch f.Type {
case protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile, protocol.FileInfoTypeSymlinkUnknown:
return true
default:
return false
}
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return !f.NoPermissions
}
func (f FileInfoTruncated) FileSize() int64 {
if f.IsDirectory() || f.IsDeleted() {
return 128
}
return f.Size
}
func (f FileInfoTruncated) FileName() string {
return f.Name
}

914
lib/db/structs.pb.go Normal file
View File

@@ -0,0 +1,914 @@
// Code generated by protoc-gen-gogo.
// source: structs.proto
// DO NOT EDIT!
/*
Package db is a generated protocol buffer package.
It is generated from these files:
structs.proto
It has these top-level messages:
FileVersion
VersionList
FileInfoTruncated
*/
package db
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import protocol "github.com/syncthing/syncthing/lib/protocol"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
const _ = proto.GoGoProtoPackageIsVersion1
type FileVersion struct {
Version protocol.Vector `protobuf:"bytes,1,opt,name=version" json:"version"`
Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
}
func (m *FileVersion) Reset() { *m = FileVersion{} }
func (m *FileVersion) String() string { return proto.CompactTextString(m) }
func (*FileVersion) ProtoMessage() {}
func (*FileVersion) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{0} }
type VersionList struct {
Versions []FileVersion `protobuf:"bytes,1,rep,name=versions" json:"versions"`
}
func (m *VersionList) Reset() { *m = VersionList{} }
func (*VersionList) ProtoMessage() {}
func (*VersionList) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{1} }
// Must be the same as FileInfo but without the blocks field
type FileInfoTruncated struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Type protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
Modified int64 `protobuf:"varint,5,opt,name=modified,proto3" json:"modified,omitempty"`
Deleted bool `protobuf:"varint,6,opt,name=deleted,proto3" json:"deleted,omitempty"`
Invalid bool `protobuf:"varint,7,opt,name=invalid,proto3" json:"invalid,omitempty"`
NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
Version protocol.Vector `protobuf:"bytes,9,opt,name=version" json:"version"`
LocalVersion int64 `protobuf:"varint,10,opt,name=local_version,json=localVersion,proto3" json:"local_version,omitempty"`
}
func (m *FileInfoTruncated) Reset() { *m = FileInfoTruncated{} }
func (*FileInfoTruncated) ProtoMessage() {}
func (*FileInfoTruncated) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{2} }
func init() {
proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
proto.RegisterType((*VersionList)(nil), "db.VersionList")
proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
}
func (m *FileVersion) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *FileVersion) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
n1, err := m.Version.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n1
if len(m.Device) > 0 {
data[i] = 0x12
i++
i = encodeVarintStructs(data, i, uint64(len(m.Device)))
i += copy(data[i:], m.Device)
}
return i, nil
}
func (m *VersionList) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *VersionList) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Versions) > 0 {
for _, msg := range m.Versions {
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(msg.ProtoSize()))
n, err := msg.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n
}
}
return i, nil
}
func (m *FileInfoTruncated) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *FileInfoTruncated) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Name) > 0 {
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(len(m.Name)))
i += copy(data[i:], m.Name)
}
if m.Type != 0 {
data[i] = 0x10
i++
i = encodeVarintStructs(data, i, uint64(m.Type))
}
if m.Size != 0 {
data[i] = 0x18
i++
i = encodeVarintStructs(data, i, uint64(m.Size))
}
if m.Permissions != 0 {
data[i] = 0x20
i++
i = encodeVarintStructs(data, i, uint64(m.Permissions))
}
if m.Modified != 0 {
data[i] = 0x28
i++
i = encodeVarintStructs(data, i, uint64(m.Modified))
}
if m.Deleted {
data[i] = 0x30
i++
if m.Deleted {
data[i] = 1
} else {
data[i] = 0
}
i++
}
if m.Invalid {
data[i] = 0x38
i++
if m.Invalid {
data[i] = 1
} else {
data[i] = 0
}
i++
}
if m.NoPermissions {
data[i] = 0x40
i++
if m.NoPermissions {
data[i] = 1
} else {
data[i] = 0
}
i++
}
data[i] = 0x4a
i++
i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
n2, err := m.Version.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n2
if m.LocalVersion != 0 {
data[i] = 0x50
i++
i = encodeVarintStructs(data, i, uint64(m.LocalVersion))
}
return i, nil
}
func encodeFixed64Structs(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Structs(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintStructs(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *FileVersion) ProtoSize() (n int) {
var l int
_ = l
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
l = len(m.Device)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
return n
}
func (m *VersionList) ProtoSize() (n int) {
var l int
_ = l
if len(m.Versions) > 0 {
for _, e := range m.Versions {
l = e.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
}
}
return n
}
func (m *FileInfoTruncated) ProtoSize() (n int) {
var l int
_ = l
l = len(m.Name)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
if m.Type != 0 {
n += 1 + sovStructs(uint64(m.Type))
}
if m.Size != 0 {
n += 1 + sovStructs(uint64(m.Size))
}
if m.Permissions != 0 {
n += 1 + sovStructs(uint64(m.Permissions))
}
if m.Modified != 0 {
n += 1 + sovStructs(uint64(m.Modified))
}
if m.Deleted {
n += 2
}
if m.Invalid {
n += 2
}
if m.NoPermissions {
n += 2
}
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
if m.LocalVersion != 0 {
n += 1 + sovStructs(uint64(m.LocalVersion))
}
return n
}
func sovStructs(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozStructs(x uint64) (n int) {
return sovStructs(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *FileVersion) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: FileVersion: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: FileVersion: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Device = append(m.Device[:0], data[iNdEx:postIndex]...)
if m.Device == nil {
m.Device = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *VersionList) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: VersionList: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: VersionList: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Versions = append(m.Versions, FileVersion{})
if err := m.Versions[len(m.Versions)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *FileInfoTruncated) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: FileInfoTruncated: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: FileInfoTruncated: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Name = string(data[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
}
m.Type = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Type |= (protocol.FileInfoType(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType)
}
m.Size = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Size |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Permissions", wireType)
}
m.Permissions = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Permissions |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Modified", wireType)
}
m.Modified = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Modified |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Deleted = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Invalid = bool(v != 0)
case 8:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field NoPermissions", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.NoPermissions = bool(v != 0)
case 9:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalVersion", wireType)
}
m.LocalVersion = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.LocalVersion |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipStructs(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthStructs
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipStructs(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthStructs = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowStructs = fmt.Errorf("proto: integer overflow")
)
var fileDescriptorStructs = []byte{
// 401 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x4f, 0xcb, 0xd3, 0x30,
0x1c, 0x6e, 0xb7, 0xba, 0xf5, 0x4d, 0xdf, 0x4e, 0x0d, 0x32, 0xca, 0x0e, 0xdd, 0x98, 0x08, 0x22,
0xd8, 0xe9, 0xc4, 0x8b, 0xc7, 0x1d, 0x06, 0x82, 0x07, 0x29, 0x32, 0x8f, 0xa3, 0x4d, 0xb2, 0x2e,
0xd0, 0x26, 0xa5, 0x49, 0x07, 0xf3, 0x93, 0x78, 0xdc, 0xc7, 0xd9, 0xd1, 0x2f, 0xa0, 0xe8, 0xfc,
0x22, 0x66, 0x49, 0x3b, 0x7b, 0x7c, 0x0f, 0x81, 0xdf, 0x93, 0xe7, 0xcf, 0xef, 0x21, 0x01, 0xbe,
0x90, 0x55, 0x8d, 0xa4, 0x88, 0xca, 0x8a, 0x4b, 0x0e, 0x7b, 0x38, 0x9d, 0xbc, 0xce, 0xa8, 0xdc,
0xd7, 0x69, 0x84, 0x78, 0xb1, 0xc8, 0x78, 0xc6, 0x17, 0x9a, 0x4a, 0xeb, 0x9d, 0x46, 0x1a, 0xe8,
0xc9, 0x58, 0x26, 0xef, 0x3b, 0x72, 0x71, 0x64, 0x48, 0xee, 0x29, 0xcb, 0x3a, 0x53, 0x4e, 0x53,
0x93, 0x80, 0x78, 0xbe, 0x48, 0x49, 0x69, 0x6c, 0xf3, 0xaf, 0xc0, 0x5b, 0xd3, 0x9c, 0x6c, 0x48,
0x25, 0x28, 0x67, 0xf0, 0x0d, 0x18, 0x1e, 0xcc, 0x18, 0xd8, 0x33, 0xfb, 0xa5, 0xb7, 0x7c, 0x12,
0xb5, 0xa6, 0x68, 0x43, 0x90, 0xe4, 0xd5, 0xca, 0x39, 0xff, 0x9a, 0x5a, 0x71, 0x2b, 0x83, 0x63,
0x30, 0xc0, 0xe4, 0x40, 0x11, 0x09, 0x7a, 0xca, 0x70, 0x1f, 0x37, 0x68, 0xbe, 0x06, 0x5e, 0x13,
0xfa, 0x89, 0x0a, 0x09, 0xdf, 0x02, 0xb7, 0x71, 0x08, 0x95, 0xdc, 0x57, 0xc9, 0x8f, 0x23, 0x9c,
0x46, 0x9d, 0xdd, 0x4d, 0xf0, 0x4d, 0xf6, 0xc1, 0xf9, 0x7e, 0x9a, 0x5a, 0xf3, 0x9f, 0x3d, 0xf0,
0xf4, 0xaa, 0xfa, 0xc8, 0x76, 0xfc, 0x4b, 0x55, 0x33, 0x94, 0x48, 0x82, 0x21, 0x04, 0x0e, 0x4b,
0x0a, 0xa2, 0x4b, 0xde, 0xc5, 0x7a, 0x86, 0xaf, 0x80, 0x23, 0x8f, 0xa5, 0xe9, 0x31, 0x5a, 0x8e,
0xff, 0x17, 0xbf, 0xd9, 0x15, 0x1b, 0x6b, 0xcd, 0xd5, 0x2f, 0xe8, 0x37, 0x12, 0xf4, 0x95, 0xb6,
0x1f, 0xeb, 0x19, 0xce, 0x80, 0x57, 0x92, 0xaa, 0xa0, 0xc2, 0xb4, 0x74, 0x14, 0xe5, 0xc7, 0xdd,
0x2b, 0x38, 0x01, 0x6e, 0xc1, 0x31, 0xdd, 0x51, 0x82, 0x83, 0x47, 0xda, 0x79, 0xc3, 0x30, 0x00,
0x43, 0x4c, 0x72, 0xa2, 0xca, 0x05, 0x03, 0x45, 0xb9, 0x71, 0x0b, 0xaf, 0x0c, 0x65, 0x87, 0x24,
0xa7, 0x38, 0x18, 0x1a, 0xa6, 0x81, 0xf0, 0x05, 0x18, 0x31, 0xbe, 0xed, 0x2e, 0x75, 0xb5, 0xc0,
0x67, 0xfc, 0x73, 0x67, 0x6d, 0xe7, 0x53, 0xee, 0x1e, 0xf6, 0x29, 0xcf, 0x81, 0x9f, 0x73, 0x94,
0xe4, 0xdb, 0xd6, 0x07, 0x74, 0xdb, 0x7b, 0x7d, 0xd9, 0xbc, 0xb7, 0x79, 0xdf, 0xd5, 0xb3, 0xf3,
0x9f, 0xd0, 0x3a, 0x5f, 0x42, 0xfb, 0x87, 0x3a, 0xbf, 0x2f, 0xa1, 0x75, 0xfa, 0x1b, 0xda, 0xe9,
0x40, 0x2f, 0x78, 0xf7, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xb1, 0x7f, 0x07, 0x98, 0x02, 0x00,
0x00,
}

35
lib/db/structs.proto Normal file
View File

@@ -0,0 +1,35 @@
syntax = "proto3";
package db;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "github.com/syncthing/syncthing/lib/protocol/bep.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;
message FileVersion {
protocol.Vector version = 1 [(gogoproto.nullable) = false];
bytes device = 2;
}
message VersionList {
option (gogoproto.goproto_stringer) = false;
repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
}
// Must be the same as FileInfo but without the blocks field
message FileInfoTruncated {
option (gogoproto.goproto_stringer) = false;
string name = 1;
protocol.FileInfoType type = 2;
int64 size = 3;
uint32 permissions = 4;
int64 modified = 5;
bool deleted = 6;
bool invalid = 7;
bool no_permissions = 8;
protocol.Vector version = 9 [(gogoproto.nullable) = false];
int64 local_version = 10;
}

BIN
lib/db/testdata/oldformat.db.zip vendored
View File

Binary file not shown.

52
lib/db/truncated.go
View File

@@ -1,52 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"github.com/calmh/xdr"
"github.com/syncthing/syncthing/lib/protocol"
)
type FileInfoTruncated struct {
protocol.FileInfo
}
func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
return o.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: bs})
}
func (o *FileInfoTruncated) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Name = u.UnmarshalStringMax(8192)
o.Flags = u.UnmarshalUint32()
o.Modified = int64(u.UnmarshalUint64())
(&o.Version).UnmarshalXDRFrom(u)
o.LocalVersion = int64(u.UnmarshalUint64())
_BlocksSize := int(u.UnmarshalUint32())
if _BlocksSize < 0 {
return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
} else if _BlocksSize == 0 {
o.Blocks = nil
} else {
if _BlocksSize > 10000000 {
return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
}
for i := 0; i < _BlocksSize; i++ {
size := int64(u.UnmarshalUint32())
o.CachedSize += size
u.UnmarshalBytes()
}
}
return u.Error
}
func BlocksToSize(num int) int64 {
if num < 2 {
return protocol.BlockSize / 2
}
return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
}

72
lib/discover/local.go
View File

@@ -4,10 +4,14 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate go run ../../script/protofmt.go local.proto
//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. local.proto
package discover
import (
"bytes"
"encoding/binary"
"encoding/hex"
"io"
"net"
@@ -38,6 +42,8 @@ type localClient struct {
const (
BroadcastInterval = 30 * time.Second
CacheLifeTime = 3 * BroadcastInterval
Magic = uint32(0x2EA7D90B) // same as in BEP
v13Magic = uint32(0x7D79BC40) // previous version
)
func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (FinderService, error) {
@@ -107,25 +113,19 @@ func (c *localClient) Error() error {
}
func (c *localClient) announcementPkt() Announce {
var addrs []Address
for _, addr := range c.addrList.AllAddresses() {
addrs = append(addrs, Address{
URL: addr,
})
}
return Announce{
Magic: AnnouncementMagic,
This: Device{
ID: c.myID[:],
Addresses: addrs,
},
ID: c.myID[:],
Addresses: c.addrList.AllAddresses(),
}
}
func (c *localClient) sendLocalAnnouncements() {
msg := make([]byte, 4)
binary.BigEndian.PutUint32(msg, Magic)
var pkt = c.announcementPkt()
msg := pkt.MustMarshalXDR()
bs, _ := pkt.Marshal()
msg = append(msg, bs...)
for {
c.beacon.Send(msg)
@@ -138,26 +138,44 @@ func (c *localClient) sendLocalAnnouncements() {
}
func (c *localClient) recvAnnouncements(b beacon.Interface) {
warnedAbout := make(map[string]bool)
for {
buf, addr := b.Recv()
if len(buf) < 4 {
l.Debugf("discover: short packet from %s")
continue
}
magic := binary.BigEndian.Uint32(buf)
switch magic {
case Magic:
// All good
case v13Magic:
// Old version
if !warnedAbout[addr.String()] {
l.Warnf("Incompatible (v0.13) local discovery packet from %v - upgrade that device to connect", addr)
warnedAbout[addr.String()] = true
}
continue
default:
l.Debugf("discover: Incorrect magic %x from %s", magic, addr)
continue
}
var pkt Announce
err := pkt.UnmarshalXDR(buf)
err := pkt.Unmarshal(buf[4:])
if err != nil && err != io.EOF {
l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
continue
}
if pkt.Magic != AnnouncementMagic {
l.Debugf("discover: Incorrect magic from %s: %s != %s", addr, pkt.Magic, AnnouncementMagic)
continue
}
l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.ID))
var newDevice bool
if !bytes.Equal(pkt.This.ID, c.myID[:]) {
newDevice = c.registerDevice(addr, pkt.This)
if !bytes.Equal(pkt.ID, c.myID[:]) {
newDevice = c.registerDevice(addr, pkt)
}
if newDevice {
@@ -171,7 +189,7 @@ func (c *localClient) recvAnnouncements(b beacon.Interface) {
}
}
func (c *localClient) registerDevice(src net.Addr, device Device) bool {
func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
var id protocol.DeviceID
copy(id[:], device.ID)
@@ -186,7 +204,7 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
l.Debugln("discover: Registering addresses for", id)
var validAddresses []string
for _, addr := range device.Addresses {
u, err := url.Parse(addr.URL)
u, err := url.Parse(addr)
if err != nil {
continue
}
@@ -219,10 +237,10 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
l.Debugf("discover: Reconstructed URL is %#v", u)
validAddresses = append(validAddresses, u.String())
l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr.URL, u.String())
l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr, u.String())
} else {
validAddresses = append(validAddresses, addr.URL)
l.Debugf("discover: Accepted address %s verbatim", addr.URL)
validAddresses = append(validAddresses, addr)
l.Debugf("discover: Accepted address %s verbatim", addr)
}
}

368
lib/discover/local.pb.go Normal file
View File

@@ -0,0 +1,368 @@
// Code generated by protoc-gen-gogo.
// source: local.proto
// DO NOT EDIT!
/*
Package discover is a generated protocol buffer package.
It is generated from these files:
local.proto
It has these top-level messages:
Announce
*/
package discover
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
const _ = proto.GoGoProtoPackageIsVersion1
type Announce struct {
ID []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
Addresses []string `protobuf:"bytes,2,rep,name=addresses" json:"addresses,omitempty"`
}
func (m *Announce) Reset() { *m = Announce{} }
func (m *Announce) String() string { return proto.CompactTextString(m) }
func (*Announce) ProtoMessage() {}
func (*Announce) Descriptor() ([]byte, []int) { return fileDescriptorLocal, []int{0} }
func init() {
proto.RegisterType((*Announce)(nil), "discover.Announce")
}
func (m *Announce) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Announce) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.ID) > 0 {
data[i] = 0xa
i++
i = encodeVarintLocal(data, i, uint64(len(m.ID)))
i += copy(data[i:], m.ID)
}
if len(m.Addresses) > 0 {
for _, s := range m.Addresses {
data[i] = 0x12
i++
l = len(s)
for l >= 1<<7 {
data[i] = uint8(uint64(l)&0x7f | 0x80)
l >>= 7
i++
}
data[i] = uint8(l)
i++
i += copy(data[i:], s)
}
}
return i, nil
}
func encodeFixed64Local(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Local(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintLocal(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *Announce) ProtoSize() (n int) {
var l int
_ = l
l = len(m.ID)
if l > 0 {
n += 1 + l + sovLocal(uint64(l))
}
if len(m.Addresses) > 0 {
for _, s := range m.Addresses {
l = len(s)
n += 1 + l + sovLocal(uint64(l))
}
}
return n
}
func sovLocal(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozLocal(x uint64) (n int) {
return sovLocal(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Announce) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Announce: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Announce: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthLocal
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ID = append(m.ID[:0], data[iNdEx:postIndex]...)
if m.ID == nil {
m.ID = []byte{}
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowLocal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthLocal
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, string(data[iNdEx:postIndex]))
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipLocal(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthLocal
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipLocal(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocal
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocal
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocal
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthLocal
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowLocal
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipLocal(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthLocal = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowLocal = fmt.Errorf("proto: integer overflow")
)
var fileDescriptorLocal = []byte{
// 161 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
0xc9, 0x81, 0x8b, 0xc3, 0x31, 0x2f, 0x2f, 0xbf, 0x34, 0x2f, 0x39, 0x55, 0x48, 0x8c, 0x8b, 0x29,
0x33, 0x45, 0x82, 0x51, 0x81, 0x51, 0x83, 0xc7, 0x89, 0xed, 0xd1, 0x3d, 0x79, 0x26, 0x4f, 0x97,
0x20, 0xa0, 0x88, 0x90, 0x0c, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71, 0x6a, 0xb1,
0x04, 0x93, 0x02, 0xb3, 0x06, 0x67, 0x10, 0x42, 0xc0, 0x49, 0xe4, 0xc4, 0x43, 0x39, 0x86, 0x13,
0x8f, 0xe4, 0x18, 0x2f, 0x00, 0xf1, 0x83, 0x47, 0x72, 0x0c, 0x0b, 0x1e, 0xcb, 0x31, 0x26, 0xb1,
0x81, 0x8d, 0x37, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc9, 0xec, 0xea, 0xbc, 0xa6, 0x00, 0x00,
0x00,
}

14
lib/discover/local.proto Normal file
View File

@@ -0,0 +1,14 @@
syntax = "proto3";
package discover;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;
message Announce {
bytes id = 1 [(gogoproto.customname) = "ID"];
repeated string addresses = 2;
}

29
lib/discover/localpackets.go
View File

@@ -1,29 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o localpackets_xdr.go localpackets.go
package discover
const (
AnnouncementMagic = 0x7D79BC40
)
type Announce struct {
Magic uint32
This Device
Extra []Device // max:16
}
type Device struct {
ID []byte // max:32
Addresses []Address // max:16
}
type Address struct {
URL string // max:2083
}

246
lib/discover/localpackets_xdr.go
View File

@@ -1,246 +0,0 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package discover
import (
"github.com/calmh/xdr"
)
/*
Announce Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Magic |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Device Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Extra |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more Device Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct Announce {
unsigned int Magic;
Device This;
Device Extra<16>;
}
*/
func (o Announce) XDRSize() int {
return 4 +
o.This.XDRSize() +
4 + xdr.SizeOfSlice(o.Extra)
}
func (o Announce) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o Announce) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o Announce) MarshalXDRInto(m *xdr.Marshaller) error {
m.MarshalUint32(o.Magic)
if err := o.This.MarshalXDRInto(m); err != nil {
return err
}
if l := len(o.Extra); l > 16 {
return xdr.ElementSizeExceeded("Extra", l, 16)
}
m.MarshalUint32(uint32(len(o.Extra)))
for i := range o.Extra {
if err := o.Extra[i].MarshalXDRInto(m); err != nil {
return err
}
}
return m.Error
}
func (o *Announce) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *Announce) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Magic = u.UnmarshalUint32()
(&o.This).UnmarshalXDRFrom(u)
_ExtraSize := int(u.UnmarshalUint32())
if _ExtraSize < 0 {
return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
} else if _ExtraSize == 0 {
o.Extra = nil
} else {
if _ExtraSize > 16 {
return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
}
if _ExtraSize <= len(o.Extra) {
o.Extra = o.Extra[:_ExtraSize]
} else {
o.Extra = make([]Device, _ExtraSize)
}
for i := range o.Extra {
(&o.Extra[i]).UnmarshalXDRFrom(u)
}
}
return u.Error
}
/*
Device Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ ID (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of Addresses |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more Address Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct Device {
opaque ID<32>;
Address Addresses<16>;
}
*/
func (o Device) XDRSize() int {
return 4 + len(o.ID) + xdr.Padding(len(o.ID)) +
4 + xdr.SizeOfSlice(o.Addresses)
}
func (o Device) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o Device) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o Device) MarshalXDRInto(m *xdr.Marshaller) error {
if l := len(o.ID); l > 32 {
return xdr.ElementSizeExceeded("ID", l, 32)
}
m.MarshalBytes(o.ID)
if l := len(o.Addresses); l > 16 {
return xdr.ElementSizeExceeded("Addresses", l, 16)
}
m.MarshalUint32(uint32(len(o.Addresses)))
for i := range o.Addresses {
if err := o.Addresses[i].MarshalXDRInto(m); err != nil {
return err
}
}
return m.Error
}
func (o *Device) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *Device) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.ID = u.UnmarshalBytesMax(32)
_AddressesSize := int(u.UnmarshalUint32())
if _AddressesSize < 0 {
return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
} else if _AddressesSize == 0 {
o.Addresses = nil
} else {
if _AddressesSize > 16 {
return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
}
if _AddressesSize <= len(o.Addresses) {
o.Addresses = o.Addresses[:_AddressesSize]
} else {
o.Addresses = make([]Address, _AddressesSize)
}
for i := range o.Addresses {
(&o.Addresses[i]).UnmarshalXDRFrom(u)
}
}
return u.Error
}
/*
Address Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ URL (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct Address {
string URL<2083>;
}
*/
func (o Address) XDRSize() int {
return 4 + len(o.URL) + xdr.Padding(len(o.URL))
}
func (o Address) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o Address) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o Address) MarshalXDRInto(m *xdr.Marshaller) error {
if l := len(o.URL); l > 2083 {
return xdr.ElementSizeExceeded("URL", l, 2083)
}
m.MarshalString(o.URL)
return m.Error
}
func (o *Address) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *Address) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.URL = u.UnmarshalStringMax(2083)
return u.Error
}

155
lib/model/model.go
View File

@@ -40,10 +40,8 @@ import (
// How many files to send in each Index/IndexUpdate message.
const (
indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
indexPerFileSize = 250 // Each FileInfo is approximately this big, in bytes, excluding BlockInfos
indexPerBlockSize = 40 // Each BlockInfo is approximately this big
indexBatchSize = 1000 // Either way, don't include more files than this
indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
indexBatchSize = 1000 // Either way, don't include more files than this
)
type service interface {
@@ -95,7 +93,7 @@ type Model struct {
conn map[protocol.DeviceID]connections.Connection
helloMessages map[protocol.DeviceID]protocol.HelloResult
deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfigMessage
deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfig
devicePaused map[protocol.DeviceID]bool
deviceDownloads map[protocol.DeviceID]*deviceDownloadState
pmut sync.RWMutex // protects the above
@@ -149,7 +147,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
folderStatRefs: make(map[string]*stats.FolderStatisticsReference),
conn: make(map[protocol.DeviceID]connections.Connection),
helloMessages: make(map[protocol.DeviceID]protocol.HelloResult),
deviceClusterConf: make(map[protocol.DeviceID]protocol.ClusterConfigMessage),
deviceClusterConf: make(map[protocol.DeviceID]protocol.ClusterConfig),
devicePaused: make(map[protocol.DeviceID]bool),
deviceDownloads: make(map[protocol.DeviceID]*deviceDownloadState),
fmut: sync.NewRWMutex(),
@@ -414,7 +412,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
// This might might be more than it really is, because some blocks can be of a smaller size.
downloaded = int64(counts[ft.Name] * protocol.BlockSize)
fileNeed = ft.Size() - downloaded
fileNeed = ft.Size - downloaded
if fileNeed < 0 {
fileNeed = 0
}
@@ -436,7 +434,7 @@ func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
} else {
deleted++
}
bytes += f.Size()
bytes += f.FileSize()
return
}
@@ -548,12 +546,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
// Index is called when a new device is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
if flags != 0 {
l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
return
}
func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
if !m.folderSharedWith(folder, deviceID) {
@@ -595,12 +588,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
// IndexUpdate is called for incremental updates to connected devices' indexes.
// Implements the protocol.Model interface.
func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
if flags != 0 {
l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
return
}
func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
if !m.folderSharedWith(folder, deviceID) {
@@ -651,7 +639,7 @@ func (m *Model) folderSharedWithUnlocked(folder string, deviceID protocol.Device
return false
}
func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfigMessage) {
func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
// Check the peer device's announced folders against our own. Emits events
// for folders that we don't expect (unknown or not shared).
// Also, collect a list of folders we do share, and if he's interested in
@@ -659,19 +647,9 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
tempIndexFolders := make([]string, 0, len(cm.Folders))
m.fmut.Lock()
for _, folder := range cm.Folders {
if folder.Flags&^protocol.FlagFolderAll != 0 {
// There are flags set that we don't know what they mean. Fatal!
l.Warnf("Device %v: unknown flags for folder %s", deviceID, folder.ID)
m.fmut.Unlock()
m.Close(deviceID, fmt.Errorf("Unknown folder flags from device %v", deviceID))
return
}
m.fmut.Lock()
shared := m.folderSharedWithUnlocked(folder.ID, deviceID)
m.fmut.Unlock()
if !shared {
if !m.folderSharedWithUnlocked(folder.ID, deviceID) {
events.Default.Log(events.FolderRejected, map[string]string{
"folder": folder.ID,
"folderLabel": folder.Label,
@@ -680,10 +658,11 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
l.Infof("Unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.ID, deviceID)
continue
}
if folder.Flags&protocol.FlagFolderDisabledTempIndexes == 0 {
if !folder.DisableTempIndexes {
tempIndexFolders = append(tempIndexFolders, folder.ID)
}
}
m.fmut.Unlock()
// This breaks if we send multiple CM messages during the same connection.
if len(tempIndexFolders) > 0 {
@@ -733,7 +712,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
}
// The introducers' introducers are also our introducers.
if device.Flags&protocol.FlagIntroducer != 0 {
if device.Introducer {
l.Infof("Device %v is now also an introducer", id)
newDeviceCfg.Introducer = true
}
@@ -804,7 +783,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []protocol.Option, buf []byte) error {
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
if offset < 0 {
return protocol.ErrInvalid
}
@@ -813,14 +792,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
return protocol.ErrNoSuchFile
}
if flags != 0 && flags != protocol.FlagFromTemporary {
// We currently support only no flags, or FromTemporary flag.
return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
}
if deviceID != protocol.LocalDeviceID {
l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d f=%d", m, deviceID, folder, name, offset, len(buf), flags)
l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, len(buf), fromTemporary)
}
m.fmut.RLock()
folderCfg := m.folderCfgs[folder]
@@ -880,7 +853,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// Only check temp files if the flag is set, and if we are set to advertise
// the temp indexes.
if flags&protocol.FlagFromTemporary != 0 && !folderCfg.DisableTempIndexes {
if fromTemporary && !folderCfg.DisableTempIndexes {
tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
return nil
@@ -1027,8 +1000,8 @@ func (m *Model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
}
// GetHello is called when we are about to connect to some remote device.
func (m *Model) GetHello(protocol.DeviceID) protocol.Version13HelloMessage {
return protocol.Version13HelloMessage{
func (m *Model) GetHello(protocol.DeviceID) protocol.HelloIntf {
return &protocol.Hello{
DeviceName: m.deviceName,
ClientName: m.clientName,
ClientVersion: m.clientVersion,
@@ -1101,7 +1074,7 @@ func (m *Model) PauseDevice(device protocol.DeviceID) {
events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
}
func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
if !m.folderSharedWith(folder, device) {
return
}
@@ -1238,13 +1211,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
if initial {
if err = conn.Index(folder, batch, 0, nil); err != nil {
if err = conn.Index(folder, batch); err != nil {
return false
}
l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
initial = false
} else {
if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
if err = conn.IndexUpdate(folder, batch); err != nil {
return false
}
l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
@@ -1255,17 +1228,17 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
}
batch = append(batch, f)
currentBatchSize += indexPerFileSize + len(f.Blocks)*indexPerBlockSize
currentBatchSize += f.ProtoSize()
return true
})
if initial && err == nil {
err = conn.Index(folder, batch, 0, nil)
err = conn.Index(folder, batch)
if err == nil {
l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
}
} else if len(batch) > 0 && err == nil {
err = conn.IndexUpdate(folder, batch, 0, nil)
err = conn.IndexUpdate(folder, batch)
if err == nil {
l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
}
@@ -1320,11 +1293,14 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
objType := "file"
action := "modified"
// If our local vector is verison 1 AND it is the only version vector so far seen for this file then
// it is a new file. Else if it is > 1 it's not new, and if it is 1 but another shortId version vector
// exists then it is new for us but created elsewhere so the file is still not new but modified by us.
// Only if it is truly new do we change this to 'added', else we leave it as 'modified'.
if len(file.Version) == 1 && file.Version[0].Value == 1 {
// If our local vector is verison 1 AND it is the only version
// vector so far seen for this file then it is a new file. Else if
// it is > 1 it's not new, and if it is 1 but another shortId
// version vector exists then it is new for us but created elsewhere
// so the file is still not new but modified by us. Only if it is
// truly new do we change this to 'added', else we leave it as
// 'modified'.
if len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1 {
action = "added"
}
@@ -1579,10 +1555,14 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
// File has been ignored or an unsupported symlink. Set invalid bit.
l.Debugln("setting invalid bit on ignored", f)
nf := protocol.FileInfo{
Name: f.Name,
Flags: f.Flags | protocol.FlagInvalid,
Modified: f.Modified,
Version: f.Version, // The file is still the same, so don't bump version
Name: f.Name,
Type: f.Type,
Size: f.Size,
Modified: f.Modified,
Permissions: f.Permissions,
NoPermissions: f.NoPermissions,
Invalid: true,
Version: f.Version, // The file is still the same, so don't bump version
}
batch = append(batch, nf)
} else if _, err := osutil.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
@@ -1597,16 +1577,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
nf := protocol.FileInfo{
Name: f.Name,
Flags: f.Flags | protocol.FlagDeleted,
Type: f.Type,
Size: f.Size,
Modified: f.Modified,
Deleted: true,
Version: f.Version.Update(m.shortID),
}
// The deleted file might have been ignored at some
// point, but it currently isn't so we make sure to
// clear the invalid bit.
nf.Flags &^= protocol.FlagInvalid
batch = append(batch, nf)
}
}
@@ -1672,30 +1649,21 @@ func (m *Model) numHashers(folder string) int {
// generateClusterConfig returns a ClusterConfigMessage that is correct for
// the given peer device
func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfigMessage {
var message protocol.ClusterConfigMessage
func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfig {
var message protocol.ClusterConfig
m.fmut.RLock()
for _, folder := range m.deviceFolders[device] {
folderCfg := m.cfg.Folders()[folder]
protocolFolder := protocol.Folder{
ID: folder,
Label: folderCfg.Label,
ID: folder,
Label: folderCfg.Label,
ReadOnly: folderCfg.Type == config.FolderTypeReadOnly,
IgnorePermissions: folderCfg.IgnorePerms,
IgnoreDelete: folderCfg.IgnoreDelete,
DisableTempIndexes: folderCfg.DisableTempIndexes,
}
var flags uint32
if folderCfg.Type == config.FolderTypeReadOnly {
flags |= protocol.FlagFolderReadOnly
}
if folderCfg.IgnorePerms {
flags |= protocol.FlagFolderIgnorePerms
}
if folderCfg.IgnoreDelete {
flags |= protocol.FlagFolderIgnoreDelete
}
if folderCfg.DisableTempIndexes {
flags |= protocol.FlagFolderDisabledTempIndexes
}
protocolFolder.Flags = flags
for _, device := range m.folderDevices[folder] {
// DeviceID is a value type, but with an underlying array. Copy it
// so we don't grab aliases to the same array later on in device[:]
@@ -1707,14 +1675,11 @@ func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.Cluster
ID: device[:],
Name: deviceCfg.Name,
Addresses: deviceCfg.Addresses,
Compression: uint32(deviceCfg.Compression),
Compression: deviceCfg.Compression,
CertName: deviceCfg.CertName,
Flags: protocol.FlagShareTrusted,
Introducer: deviceCfg.Introducer,
}
if deviceCfg.Introducer {
protocolDevice.Flags |= protocol.FlagIntroducer
}
protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
}
message.Folders = append(message.Folders, protocolFolder)
@@ -1759,7 +1724,7 @@ func (m *Model) Override(folder string) {
have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
if !ok || have.Name != need.Name {
// We are missing the file
need.Flags |= protocol.FlagDeleted
need.Deleted = true
need.Blocks = nil
need.Version = need.Version.Update(m.shortID)
} else {
@@ -1868,7 +1833,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
if !dirsonly && base != "" {
last[base] = []interface{}{
time.Unix(f.Modified, 0), f.Size(),
time.Unix(f.Modified, 0), f.FileSize(),
}
}
@@ -2181,11 +2146,7 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool, ignores *ignore.Matcher) []protocol.FileInfo {
for i := 0; i < len(fs); {
if fs[i].Flags&^protocol.FlagsAll != 0 {
l.Debugln("dropping update for file with unknown bits set", fs[i])
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]
} else if fs[i].IsDeleted() && dropDeletes {
if fs[i].IsDeleted() && dropDeletes {
l.Debugln("dropping update for undesired delete", fs[i])
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]

113
lib/model/model_test.go
View File

@@ -55,19 +55,19 @@ func init() {
var testDataExpected = map[string]protocol.FileInfo{
"foo": {
Name: "foo",
Flags: 0,
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
},
"empty": {
Name: "empty",
Flags: 0,
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
},
"bar": {
Name: "bar",
Flags: 0,
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
},
@@ -77,8 +77,9 @@ func init() {
// Fix expected test data to match reality
for n, f := range testDataExpected {
fi, _ := os.Stat("testdata/" + n)
f.Flags = uint32(fi.Mode())
f.Permissions = uint32(fi.Mode())
f.Modified = fi.ModTime().Unix()
f.Size = fi.Size()
testDataExpected[n] = f
}
}
@@ -98,7 +99,7 @@ func TestRequest(t *testing.T) {
// Existing, shared file
bs = bs[:6]
err := m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
err := m.Request(device1, "default", "foo", 0, nil, false, bs)
if err != nil {
t.Error(err)
}
@@ -107,32 +108,32 @@ func TestRequest(t *testing.T) {
}
// Existing, nonshared file
err = m.Request(device2, "default", "foo", 0, nil, 0, nil, bs)
err = m.Request(device2, "default", "foo", 0, nil, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Nonexistent file
err = m.Request(device1, "default", "nonexistent", 0, nil, 0, nil, bs)
err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Shared folder, but disallowed file name
err = m.Request(device1, "default", "../walk.go", 0, nil, 0, nil, bs)
err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Negative offset
err = m.Request(device1, "default", "foo", -4, nil, 0, nil, bs[:0])
err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Larger block than available
bs = bs[:42]
err = m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
err = m.Request(device1, "default", "foo", 0, nil, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
@@ -168,11 +169,11 @@ func benchmarkIndex(b *testing.B, nfiles int) {
m.ServeBackground()
files := genFiles(nfiles)
m.Index(device1, "default", files, 0, nil)
m.Index(device1, "default", files)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.Index(device1, "default", files, 0, nil)
m.Index(device1, "default", files)
}
b.ReportAllocs()
}
@@ -199,11 +200,11 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
files := genFiles(nfiles)
ufiles := genFiles(nufiles)
m.Index(device1, "default", files, 0, nil)
m.Index(device1, "default", files)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.IndexUpdate(device1, "default", ufiles, 0, nil)
m.IndexUpdate(device1, "default", ufiles)
}
b.ReportAllocs()
}
@@ -211,8 +212,6 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
type downloadProgressMessage struct {
folder string
updates []protocol.FileDownloadProgressUpdate
flags uint32
options []protocol.Option
}
type FakeConnection struct {
@@ -240,11 +239,11 @@ func (f FakeConnection) Option(string) string {
return ""
}
func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
func (FakeConnection) Index(string, []protocol.FileInfo) error {
return nil
}
func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
return nil
}
@@ -252,7 +251,7 @@ func (f FakeConnection) Request(folder, name string, offset int64, size int, has
return f.requestData, nil
}
func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
func (FakeConnection) ClusterConfig(protocol.ClusterConfig) {}
func (FakeConnection) Ping() bool {
return true
@@ -266,12 +265,10 @@ func (FakeConnection) Statistics() protocol.Statistics {
return protocol.Statistics{}
}
func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
folder: folder,
updates: updates,
flags: flags,
options: options,
})
}
@@ -305,7 +302,7 @@ func BenchmarkRequest(b *testing.B) {
},
Connection: fc,
}, protocol.HelloResult{})
m.Index(device1, "default", files, 0, nil)
m.Index(device1, "default", files)
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -451,13 +448,13 @@ func TestClusterConfig(t *testing.T) {
if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
t.Errorf("Incorrect device ID %x != %x", id, device1)
}
if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
if !r.Devices[0].Introducer {
t.Error("Device1 should be flagged as Introducer")
}
if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
t.Errorf("Incorrect device ID %x != %x", id, device2)
}
if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
if r.Devices[1].Introducer {
t.Error("Device2 should not be flagged as Introducer")
}
@@ -471,13 +468,13 @@ func TestClusterConfig(t *testing.T) {
if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
t.Errorf("Incorrect device ID %x != %x", id, device1)
}
if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
if !r.Devices[0].Introducer {
t.Error("Device1 should be flagged as Introducer")
}
if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
t.Errorf("Incorrect device ID %x != %x", id, device2)
}
if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
if r.Devices[1].Introducer {
t.Error("Device2 should not be flagged as Introducer")
}
}
@@ -572,44 +569,6 @@ func TestIgnores(t *testing.T) {
}
}
func TestRefuseUnknownBits(t *testing.T) {
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
m.AddFolder(defaultFolderConfig)
m.ServeBackground()
m.ScanFolder("default")
m.Index(device1, "default", []protocol.FileInfo{
{
Name: "invalid1",
Flags: (protocol.FlagsAll + 1) &^ protocol.FlagInvalid,
},
{
Name: "invalid2",
Flags: (protocol.FlagsAll + 2) &^ protocol.FlagInvalid,
},
{
Name: "invalid3",
Flags: (1 << 31) &^ protocol.FlagInvalid,
},
{
Name: "valid",
Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
},
}, 0, nil)
for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
f, ok := m.CurrentGlobalFile("default", name)
if ok || f.Name == name {
t.Error("Invalid file found or name match")
}
}
f, ok := m.CurrentGlobalFile("default", "valid")
if !ok || f.Name != "valid" {
t.Error("Valid file not found or name mismatch", ok, f)
}
}
func TestROScanRecovery(t *testing.T) {
ldb := db.OpenMemory()
set := db.NewFileSet("default", ldb)
@@ -789,17 +748,18 @@ func TestGlobalDirectoryTree(t *testing.T) {
m.ServeBackground()
b := func(isfile bool, path ...string) protocol.FileInfo {
flags := uint32(protocol.FlagDirectory)
typ := protocol.FileInfoTypeDirectory
blocks := []protocol.BlockInfo{}
if isfile {
flags = 0
typ = protocol.FileInfoTypeFile
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
}
return protocol.FileInfo{
Name: filepath.Join(path...),
Flags: flags,
Type: typ,
Modified: 0x666,
Blocks: blocks,
Size: 0xa,
}
}
@@ -871,7 +831,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
return string(bytes)
}
m.Index(device1, "default", testdata, 0, nil)
m.Index(device1, "default", testdata)
result := m.GlobalDirectoryTree("default", "", -1, false)
@@ -1039,17 +999,18 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
m.ServeBackground()
b := func(isfile bool, path ...string) protocol.FileInfo {
flags := uint32(protocol.FlagDirectory)
typ := protocol.FileInfoTypeDirectory
blocks := []protocol.BlockInfo{}
if isfile {
flags = 0
typ = protocol.FileInfoTypeFile
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
}
return protocol.FileInfo{
Name: filepath.Join(path...),
Flags: flags,
Type: typ,
Modified: 0x666,
Blocks: blocks,
Size: 0xa,
}
}
@@ -1130,7 +1091,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
return string(bytes)
}
m.Index(device1, "default", testdata, 0, nil)
m.Index(device1, "default", testdata)
result := m.GlobalDirectoryTree("default", "", -1, false)
@@ -1215,7 +1176,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
m.ScanFolder("default")
files := genDeepFiles(n1, n2)
m.Index(device1, "default", files, 0, nil)
m.Index(device1, "default", files)
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -1244,12 +1205,12 @@ func TestIgnoreDelete(t *testing.T) {
}
// Mark it for deletion
f.Flags = protocol.FlagDeleted
f.Deleted = true
f.Version = f.Version.Update(142) // arbitrary short remote ID
f.Blocks = nil
// Send the index
m.Index(device1, "default", []protocol.FileInfo{f}, 0, nil)
m.Index(device1, "default", []protocol.FileInfo{f})
// Make sure we ignored it
f, ok = m.CurrentGlobalFile("default", "foo")

2
lib/model/progressemitter.go
View File

@@ -137,7 +137,7 @@ func (t *ProgressEmitter) sendDownloadProgressMessages() {
updates := state.update(folder, activePullers)
if len(updates) > 0 {
conn.DownloadProgress(folder, updates, 0, nil)
conn.DownloadProgress(folder, updates)
}
}
}

7
lib/model/progressemitter_test.go
View File

@@ -100,7 +100,6 @@ func TestProgressEmitter(t *testing.T) {
}
func TestSendDownloadProgressMessages(t *testing.T) {
c := config.Wrap("/tmp/test", config.Configuration{})
c.SetOptions(config.OptionsConfiguration{
ProgressUpdateIntervalS: 0,
@@ -112,7 +111,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
p := NewProgressEmitter(c)
p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
expect := func(updateIdx int, state *sharedPullerState, updateType uint32, version protocol.Vector, blocks []int32, remove bool) {
expect := func(updateIdx int, state *sharedPullerState, updateType protocol.FileDownloadProgressUpdateType, version protocol.Vector, blocks []int32, remove bool) {
messageIdx := -1
for i, msg := range fc.downloadProgressMessages {
if msg.folder == state.folder {
@@ -346,7 +345,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
file: protocol.FileInfo{
Name: "state5",
Version: v1,
Flags: protocol.FlagDirectory,
Type: protocol.FileInfoTypeDirectory,
Blocks: blocks,
},
mut: sync.NewRWMutex(),
@@ -359,7 +358,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
file: protocol.FileInfo{
Name: "state6",
Version: v1,
Flags: protocol.FlagSymlink,
Type: protocol.FileInfoTypeSymlinkUnknown,
},
mut: sync.NewRWMutex(),
available: []int32{1, 2, 3},

14
lib/model/rwfolder.go
View File

@@ -163,7 +163,7 @@ func (f *rwFolder) configureCopiersAndPullers(config config.FolderConfiguration)
// set on the local host or the FlagNoPermBits has been set on the file/dir
// which is being pulled.
func (f *rwFolder) ignorePermissions(file protocol.FileInfo) bool {
return f.ignorePerms || file.Flags&protocol.FlagNoPermBits != 0
return f.ignorePerms || file.NoPermissions
}
// Serve will run scans and pulls. It will return when Stop()ed or on a
@@ -435,7 +435,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
if !handleFile(file) {
// A new or changed file or symlink. This is the only case where we
// do stuff concurrently in the background
f.queue.Push(file.Name, file.Size(), file.Modified)
f.queue.Push(file.Name, file.Size, file.Modified)
}
changed++
@@ -569,7 +569,7 @@ func (f *rwFolder) handleDir(file protocol.FileInfo) {
}()
realName := filepath.Join(f.dir, file.Name)
mode := os.FileMode(file.Flags & 0777)
mode := os.FileMode(file.Permissions & 0777)
if f.ignorePermissions(file) {
mode = 0777
}
@@ -909,7 +909,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
// touching the file. If we can't stat the file we'll just pull it.
if info, err := osutil.Lstat(realName); err == nil {
mtime := f.virtualMtimeRepo.GetMtime(file.Name, info.ModTime())
if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size() {
if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size {
l.Debugln("file modified but not rescanned; not pulling:", realName)
// Scan() is synchronous (i.e. blocks until the scan is
// completed and returns an error), but a scan can't happen
@@ -965,7 +965,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
} else {
// Copy the blocks, as we don't want to shuffle them on the FileInfo
blocks = append(blocks, file.Blocks...)
blocksSize = file.Size()
blocksSize = file.Size
}
if f.checkFreeSpace {
@@ -1021,7 +1021,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
func (f *rwFolder) shortcutFile(file protocol.FileInfo) error {
realName := filepath.Join(f.dir, file.Name)
if !f.ignorePermissions(file) {
if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil {
if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
f.newError(file.Name, err)
return err
@@ -1235,7 +1235,7 @@ func (f *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
func (f *rwFolder) performFinish(state *sharedPullerState) error {
// Set the correct permission bits on the new file
if !f.ignorePermissions(state.file) {
if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil {
if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
return err
}
}

6
lib/model/rwfolder_test.go
View File

@@ -50,10 +50,8 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
}
return protocol.FileInfo{
Name: filename,
Flags: 0,
Modified: 0,
Blocks: existingBlocks,
Name: filename,
Blocks: existingBlocks,
}
}

14
lib/model/sharedpullerstate.go
View File

@@ -12,7 +12,6 @@ import (
"path/filepath"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -150,7 +149,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
if s.sparse && !s.file.IsSymlink() {
// Truncate sets the size of the file. This creates a sparse file or a
// space reservation, depending on the underlying filesystem.
if err := fd.Truncate(s.file.Size()); err != nil {
if err := fd.Truncate(s.file.Size); err != nil {
s.failLocked("dst truncate", err)
return nil, err
}
@@ -293,8 +292,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
Pulled: s.pullTotal - s.pullNeeded,
Pulling: s.pullNeeded,
BytesTotal: db.BlocksToSize(total),
BytesDone: db.BlocksToSize(done),
BytesTotal: blocksToSize(total),
BytesDone: blocksToSize(done),
}
}
@@ -321,3 +320,10 @@ func (s *sharedPullerState) Available() []int32 {
s.mut.RUnlock()
return blocks
}
func blocksToSize(num int) int64 {
if num < 2 {
return protocol.BlockSize / 2
}
return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
}

14
lib/protocol/benchmark_test.go
View File

@@ -63,8 +63,8 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
c1.Start()
// Satisfy the assertions in the protocol by sending an initial cluster config
c0.ClusterConfig(ClusterConfigMessage{})
c1.ClusterConfig(ClusterConfigMessage{})
c0.ClusterConfig(ClusterConfig{})
c1.ClusterConfig(ClusterConfig{})
// Report some useful stats and reset the timer for the actual test
b.ReportAllocs()
@@ -164,13 +164,13 @@ func negotiateTLS(cert tls.Certificate, conn0, conn1 net.Conn) (net.Conn, net.Co
type fakeModel struct{}
func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
}
func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
}
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
// We write the offset to the end of the buffer, so the receiver
// can verify that it did in fact get some data back over the
// connection.
@@ -178,11 +178,11 @@ func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offse
return nil
}
func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
}
func (m *fakeModel) Close(deviceID DeviceID, err error) {
}
func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option) {
func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate) {
}

3999
lib/protocol/bep.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

189
lib/protocol/bep.proto Normal file
View File

@@ -0,0 +1,189 @@
// protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. message.proto
syntax = "proto3";
package protocol;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;
option (gogoproto.goproto_enum_stringer_all) = true;
option (gogoproto.goproto_enum_prefix_all) = false;
// --- Pre-auth ---
message Hello {
string device_name = 1;
string client_name = 2;
string client_version = 3;
}
// --- Header ---
message Header {
MessageType type = 1;
MessageCompression compression = 2;
}
enum MessageType {
CLUSTER_CONFIG = 0 [(gogoproto.enumvalue_customname) = "messageTypeClusterConfig"];
INDEX = 1 [(gogoproto.enumvalue_customname) = "messageTypeIndex"];
INDEX_UPDATE = 2 [(gogoproto.enumvalue_customname) = "messageTypeIndexUpdate"];
REQUEST = 3 [(gogoproto.enumvalue_customname) = "messageTypeRequest"];
RESPONSE = 4 [(gogoproto.enumvalue_customname) = "messageTypeResponse"];
DOWNLOAD_PROGRESS = 5 [(gogoproto.enumvalue_customname) = "messageTypeDownloadProgress"];
PING = 6 [(gogoproto.enumvalue_customname) = "messageTypePing"];
CLOSE = 7 [(gogoproto.enumvalue_customname) = "messageTypeClose"];
}
enum MessageCompression {
NONE = 0 [(gogoproto.enumvalue_customname) = "MessageCompressionNone"];
LZ4 = 1 [(gogoproto.enumvalue_customname) = "MessageCompressionLZ4"];
}
// --- Actual messages ---
// Cluster Config
message ClusterConfig {
repeated Folder folders = 1 [(gogoproto.nullable) = false];
}
message Folder {
string id = 1 [(gogoproto.customname) = "ID"];
string label = 2;
bool read_only = 3;
bool ignore_permissions = 4;
bool ignore_delete = 5;
bool disable_temp_indexes = 6;
repeated Device devices = 16 [(gogoproto.nullable) = false];
}
message Device {
bytes id = 1 [(gogoproto.customname) = "ID"];
string name = 2;
repeated string addresses = 3;
Compression compression = 4;
string cert_name = 5;
int64 max_local_version = 6;
bool introducer = 7;
}
enum Compression {
METADATA = 0 [(gogoproto.enumvalue_customname) = "CompressMetadata"];
NEVER = 1 [(gogoproto.enumvalue_customname) = "CompressNever"];
ALWAYS = 2 [(gogoproto.enumvalue_customname) = "CompressAlways"];
}
// Index and Index Update
message Index {
string folder = 1;
repeated FileInfo files = 2 [(gogoproto.nullable) = false];
}
message IndexUpdate {
string folder = 1;
repeated FileInfo files = 2 [(gogoproto.nullable) = false];
}
message FileInfo {
option (gogoproto.goproto_stringer) = false;
string name = 1;
FileInfoType type = 2;
int64 size = 3;
uint32 permissions = 4;
int64 modified = 5;
bool deleted = 6;
bool invalid = 7;
bool no_permissions = 8;
Vector version = 9 [(gogoproto.nullable) = false];
int64 local_version = 10;
repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false];
}
enum FileInfoType {
FILE = 0 [(gogoproto.enumvalue_customname) = "FileInfoTypeFile"];
DIRECTORY = 1 [(gogoproto.enumvalue_customname) = "FileInfoTypeDirectory"];
SYMLINK_FILE = 2 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkFile"];
SYMLINK_DIRECTORY = 3 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkDirectory"];
SYMLINK_UNKNOWN = 4 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkUnknown"];
}
message BlockInfo {
option (gogoproto.goproto_stringer) = false;
int64 offset = 1;
int32 size = 2;
bytes hash = 3;
}
message Vector {
repeated Counter counters = 1 [(gogoproto.nullable) = false];
}
message Counter {
uint64 id = 1 [(gogoproto.customname) = "ID", (gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false];
uint64 value = 2;
}
// Request
message Request {
int32 id = 1 [(gogoproto.customname) = "ID"];
string folder = 2;
string name = 3;
int64 offset = 4;
int32 size = 5;
bytes hash = 6;
bool from_temporary = 7;
}
// Response
message Response {
int32 id = 1 [(gogoproto.customname) = "ID"];
bytes data = 2;
ErrorCode code = 3;
}
enum ErrorCode {
NO_ERROR = 0 [(gogoproto.enumvalue_customname) = "ErrorCodeNoError"];
GENERIC = 1 [(gogoproto.enumvalue_customname) = "ErrorCodeGeneric"];
NO_SUCH_FILE = 2 [(gogoproto.enumvalue_customname) = "ErrorCodeNoSuchFile"];
INVALID_FILE = 3 [(gogoproto.enumvalue_customname) = "ErrorCodeInvalidFile"];
}
// DownloadProgress
message DownloadProgress {
string folder = 1;
repeated FileDownloadProgressUpdate updates = 2 [(gogoproto.nullable) = false];
}
message FileDownloadProgressUpdate {
FileDownloadProgressUpdateType update_type = 1;
string name = 2;
Vector version = 3 [(gogoproto.nullable) = false];
repeated int32 block_indexes = 4;
}
enum FileDownloadProgressUpdateType {
APPEND = 0 [(gogoproto.enumvalue_customname) = "UpdateTypeAppend"];
FORGET = 1 [(gogoproto.enumvalue_customname) = "UpdateTypeForget"];
}
// Ping
message Ping {
}
// Close
message Close {
string reason = 1;
}

96
lib/protocol/bep_extensions.go Normal file
View File

@@ -0,0 +1,96 @@
// Copyright (C) 2014 The Protocol Authors.
//go:generate go run ../../script/protofmt.go bep.proto
//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. bep.proto
package protocol
import (
"bytes"
"crypto/sha256"
"fmt"
)
var (
sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
HelloMessageMagic = uint32(0x2EA7D90B)
)
func (m Hello) Magic() uint32 {
return HelloMessageMagic
}
func (f FileInfo) String() string {
return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}",
f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks)
}
func (f FileInfo) IsDeleted() bool {
return f.Deleted
}
func (f FileInfo) IsInvalid() bool {
return f.Invalid
}
func (f FileInfo) IsDirectory() bool {
return f.Type == FileInfoTypeDirectory
}
func (f FileInfo) IsSymlink() bool {
switch f.Type {
case FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile, FileInfoTypeSymlinkUnknown:
return true
default:
return false
}
}
func (f FileInfo) HasPermissionBits() bool {
return !f.NoPermissions
}
func (f FileInfo) FileSize() int64 {
if f.IsDirectory() || f.IsDeleted() {
return 128
}
return f.Size
}
func (f FileInfo) FileName() string {
return f.Name
}
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {
// If a modification is in conflict with a delete, we pick the
// modification.
if !f.IsDeleted() && other.IsDeleted() {
return true
}
if f.IsDeleted() && !other.IsDeleted() {
return false
}
// The one with the newer modification time wins.
if f.Modified > other.Modified {
return true
}
if f.Modified < other.Modified {
return false
}
// The modification times were equal. Use the device ID in the version
// vector as tie breaker.
return f.Version.Compare(other.Version) == ConcurrentGreater
}
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
// IsEmpty returns true if the block is a full block of zeroes.
func (b BlockInfo) IsEmpty() bool {
return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
}

59
lib/protocol/bufferpool.go Normal file
View File

@@ -0,0 +1,59 @@
// Copyright (C) 2016 The Protocol Authors.
package protocol
import "sync"
type bufferPool struct {
minSize int
pool sync.Pool
}
// get returns a new buffer of the requested size
func (p *bufferPool) get(size int) []byte {
intf := p.pool.Get()
if intf == nil {
// Pool is empty, must allocate.
return p.new(size)
}
bs := intf.([]byte)
if cap(bs) < size {
// Buffer was too small, leave it for someone else and allocate.
p.put(bs)
return p.new(size)
}
return bs[:size]
}
// upgrade grows the buffer to the requested size, while attempting to reuse
// it if possible.
func (p *bufferPool) upgrade(bs []byte, size int) []byte {
if cap(bs) >= size {
// Reslicing is enough, lets go!
return bs[:size]
}
// It was too small. But it pack into the pool and try to get another
// buffer.
p.put(bs)
return p.get(size)
}
// put returns the buffer to the pool
func (p *bufferPool) put(bs []byte) {
p.pool.Put(bs)
}
// new creates a new buffer of the requested size, taking the minimum
// allocation count into account. For internal use only.
func (p *bufferPool) new(size int) []byte {
allocSize := size
if allocSize < p.minSize {
// Avoid allocating tiny buffers that we won't be able to reuse for
// anything useful.
allocSize = p.minSize
}
return make([]byte, allocSize)[:size]
}

32
lib/protocol/common_test.go
View File

@@ -5,16 +5,15 @@ package protocol
import "time"
type TestModel struct {
data []byte
folder string
name string
offset int64
size int
hash []byte
flags uint32
options []Option
closedCh chan struct{}
closedErr error
data []byte
folder string
name string
offset int64
size int
hash []byte
fromTemporary bool
closedCh chan struct{}
closedErr error
}
func newTestModel() *TestModel {
@@ -23,20 +22,19 @@ func newTestModel() *TestModel {
}
}
func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
}
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
}
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
t.folder = folder
t.name = name
t.offset = offset
t.size = len(buf)
t.hash = hash
t.flags = flags
t.options = options
t.fromTemporary = fromTemporary
copy(buf, t.data)
return nil
}
@@ -46,10 +44,10 @@ func (t *TestModel) Close(deviceID DeviceID, err error) {
close(t.closedCh)
}
func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
}
func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate, uint32, []Option) {
func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate) {
}
func (t *TestModel) closedError() error {

14
lib/protocol/compression.go
View File

@@ -4,13 +4,7 @@ package protocol
import "fmt"
type Compression int
const (
CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
CompressNever
CompressAlways
compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
)
@@ -31,14 +25,6 @@ var compressionUnmarshal = map[string]Compression{
"always": CompressAlways,
}
func (c Compression) String() string {
s, ok := compressionMarshal[c]
if !ok {
return fmt.Sprintf("unknown:%d", c)
}
return s
}
func (c Compression) GoString() string {
return fmt.Sprintf("%q", c.String())
}

4
lib/protocol/conflict_test.go
View File

@@ -8,8 +8,8 @@ func TestWinsConflict(t *testing.T) {
testcases := [][2]FileInfo{
// The first should always win over the second
{{Modified: 42}, {Modified: 41}},
{{Modified: 41}, {Modified: 42, Flags: FlagDeleted}},
{{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}},
{{Modified: 41}, {Modified: 42, Deleted: true}},
{{Modified: 41, Version: Vector{[]Counter{{42, 2}, {43, 1}}}}, {Modified: 41, Version: Vector{[]Counter{{42, 1}, {43, 2}}}}},
}
for _, tc := range testcases {

4
lib/protocol/debug.go
View File

@@ -16,7 +16,3 @@ var (
func init() {
l.SetDebug("protocol", strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all")
}
func shouldDebug() bool {
return l.ShouldDebug("protocol")
}

35
lib/protocol/errors.go
View File

@@ -6,13 +6,6 @@ import (
"errors"
)
const (
ecNoError int32 = iota
ecGeneric
ecNoSuchFile
ecInvalid
)
var (
ErrNoError error
ErrGeneric = errors.New("generic error")
@@ -20,32 +13,32 @@ var (
ErrInvalid = errors.New("file is invalid")
)
var lookupError = map[int32]error{
ecNoError: ErrNoError,
ecGeneric: ErrGeneric,
ecNoSuchFile: ErrNoSuchFile,
ecInvalid: ErrInvalid,
var lookupError = map[ErrorCode]error{
ErrorCodeNoError: ErrNoError,
ErrorCodeGeneric: ErrGeneric,
ErrorCodeNoSuchFile: ErrNoSuchFile,
ErrorCodeInvalidFile: ErrInvalid,
}
var lookupCode = map[error]int32{
ErrNoError: ecNoError,
ErrGeneric: ecGeneric,
ErrNoSuchFile: ecNoSuchFile,
ErrInvalid: ecInvalid,
var lookupCode = map[error]ErrorCode{
ErrNoError: ErrorCodeNoError,
ErrGeneric: ErrorCodeGeneric,
ErrNoSuchFile: ErrorCodeNoSuchFile,
ErrInvalid: ErrorCodeInvalidFile,
}
func codeToError(errcode int32) error {
err, ok := lookupError[errcode]
func codeToError(code ErrorCode) error {
err, ok := lookupError[code]
if !ok {
return ErrGeneric
}
return err
}
func errorToCode(err error) int32 {
func errorToCode(err error) ErrorCode {
code, ok := lookupCode[err]
if !ok {
return ecGeneric
return ErrorCodeGeneric
}
return code
}

70
lib/protocol/fuzz.go
View File

@@ -1,70 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
// +build gofuzz
package protocol
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"reflect"
"sync"
)
func Fuzz(data []byte) int {
// Regenerate the length, or we'll most commonly exit quickly due to an
// unexpected eof which is unintestering.
if len(data) > 8 {
binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
}
// Setup a rawConnection we'll use to parse the message.
c := rawConnection{
cr: &countingReader{Reader: bytes.NewReader(data)},
closed: make(chan struct{}),
pool: sync.Pool{
New: func() interface{} {
return make([]byte, BlockSize)
},
},
}
// Attempt to parse the message.
hdr, msg, err := c.readMessage()
if err != nil {
return 0
}
// If parsing worked, attempt to encode it again.
newBs, err := msg.AppendXDR(nil)
if err != nil {
panic("not encodable")
}
// Create an appriate header for the re-encoding.
newMsg := make([]byte, 8)
binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
newMsg = append(newMsg, newBs...)
// Use the rawConnection to parse the re-encoding.
c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
hdr2, msg2, err := c.readMessage()
if err != nil {
fmt.Println("Initial:\n" + hex.Dump(data))
fmt.Println("New:\n" + hex.Dump(newMsg))
panic("not parseable after re-encode: " + err.Error())
}
// Make sure the data is the same as it was before.
if hdr != hdr2 {
panic("headers differ")
}
if !reflect.DeepEqual(msg, msg2) {
panic("contents differ")
}
return 1
}

89
lib/protocol/fuzz_test.go
View File

@@ -1,89 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
// +build gofuzz
package protocol
import (
"encoding/binary"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"testing/quick"
)
// This can be used to generate a corpus of valid messages as a starting point
// for the fuzzer.
func TestGenerateCorpus(t *testing.T) {
t.Skip("Use to generate initial corpus only")
n := 0
check := func(idx IndexMessage) bool {
for i := range idx.Options {
if len(idx.Options[i].Key) > 64 {
idx.Options[i].Key = idx.Options[i].Key[:64]
}
}
hdr := header{
version: 0,
msgID: 42,
msgType: messageTypeIndex,
compression: false,
}
msgBs := idx.MustMarshalXDR()
buf := make([]byte, 8)
binary.BigEndian.PutUint32(buf, encodeHeader(hdr))
binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs)))
buf = append(buf, msgBs...)
ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644)
n++
return true
}
if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil {
t.Fatal(err)
}
}
// Tests any crashers found by the fuzzer, for closer investigation.
func TestCrashers(t *testing.T) {
testFiles(t, "testdata/crashers")
}
// Tests the entire corpus, which should PASS before the fuzzer starts
// fuzzing.
func TestCorpus(t *testing.T) {
testFiles(t, "testdata/corpus")
}
func testFiles(t *testing.T, dir string) {
fd, err := os.Open(dir)
if err != nil {
t.Fatal(err)
}
crashers, err := fd.Readdirnames(-1)
if err != nil {
t.Fatal(err)
}
for _, name := range crashers {
if strings.HasSuffix(name, ".output") {
continue
}
if strings.HasSuffix(name, ".quoted") {
continue
}
t.Log(name)
crasher, err := ioutil.ReadFile(dir + "/" + name)
if err != nil {
t.Fatal(err)
}
Fuzz(crasher)
}
}

44
lib/protocol/header.go
View File

@@ -1,44 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import "github.com/calmh/xdr"
type header struct {
version int
msgID int
msgType int
compression bool
}
func (h header) MarshalXDRInto(m *xdr.Marshaller) error {
v := encodeHeader(h)
m.MarshalUint32(v)
return m.Error
}
func (h *header) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
v := u.UnmarshalUint32()
*h = decodeHeader(v)
return u.Error
}
func encodeHeader(h header) uint32 {
var isComp uint32
if h.compression {
isComp = 1 << 0 // the zeroth bit is the compression bit
}
return uint32(h.version&0xf)<<28 +
uint32(h.msgID&0xfff)<<16 +
uint32(h.msgType&0xff)<<8 +
isComp
}
func decodeHeader(u uint32) header {
return header{
version: int(u>>28) & 0xf,
msgID: int(u>>16) & 0xfff,
msgType: int(u>>8) & 0xff,
compression: u&1 == 1,
}
}

37
lib/protocol/hello.go
View File

@@ -9,10 +9,10 @@ import (
"io"
)
// The HelloMessage interface is implemented by the version specific hello
// The HelloIntf interface is implemented by the version specific hello
// message. It knows its magic number and how to serialize itself to a byte
// buffer.
type HelloMessage interface {
type HelloIntf interface {
Magic() uint32
Marshal() ([]byte, error)
}
@@ -29,12 +29,15 @@ var (
// ErrTooOldVersion12 is returned by ExchangeHello when the other side
// speaks the older, incompatible version 0.12 of the protocol.
ErrTooOldVersion12 = errors.New("the remote device speaks an older version of the protocol (v0.12) not compatible with this version")
// ErrTooOldVersion13 is returned by ExchangeHello when the other side
// speaks the older, incompatible version 0.12 of the protocol.
ErrTooOldVersion13 = errors.New("the remote device speaks an older version of the protocol (v0.13) not compatible with this version")
// ErrUnknownMagic is returned by ExchangeHellow when the other side
// speaks something entirely unknown.
ErrUnknownMagic = errors.New("the remote device speaks an unknown (newer?) version of the protocol")
)
func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
func ExchangeHello(c io.ReadWriter, h HelloIntf) (HelloResult, error) {
if err := writeHello(c, h); err != nil {
return HelloResult{}, err
}
@@ -45,7 +48,7 @@ func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
// version mismatch that we might want to alert the user about.
func IsVersionMismatch(err error) bool {
switch err {
case ErrTooOldVersion12, ErrUnknownMagic:
case ErrTooOldVersion12, ErrTooOldVersion13, ErrUnknownMagic:
return true
default:
return false
@@ -59,6 +62,28 @@ func readHello(c io.Reader) (HelloResult, error) {
}
switch binary.BigEndian.Uint32(header[:4]) {
case HelloMessageMagic:
// This is a v0.14 Hello message in proto format
msgSize := binary.BigEndian.Uint32(header[4:])
if msgSize > 1024 {
return HelloResult{}, fmt.Errorf("hello message too big")
}
buf := make([]byte, msgSize)
if _, err := io.ReadFull(c, buf); err != nil {
return HelloResult{}, err
}
var hello Hello
if err := hello.Unmarshal(buf); err != nil {
return HelloResult{}, err
}
res := HelloResult{
DeviceName: hello.DeviceName,
ClientName: hello.ClientName,
ClientVersion: hello.ClientVersion,
}
return res, nil
case Version13HelloMagic:
// This is a v0.13 Hello message in XDR format
msgSize := binary.BigEndian.Uint32(header[4:])
@@ -79,7 +104,7 @@ func readHello(c io.Reader) (HelloResult, error) {
ClientName: hello.ClientName,
ClientVersion: hello.ClientVersion,
}
return res, nil
return res, ErrTooOldVersion13
case 0x00010001, 0x00010000:
// This is the first word of a v0.12 cluster config message.
@@ -90,7 +115,7 @@ func readHello(c io.Reader) (HelloResult, error) {
return HelloResult{}, ErrUnknownMagic
}
func writeHello(c io.Writer, h HelloMessage) error {
func writeHello(c io.Writer, h HelloIntf) error {
msg, err := h.Marshal()
if err != nil {
return err

53
lib/protocol/hello_test.go
View File

@@ -13,6 +13,53 @@ import (
var spaceRe = regexp.MustCompile(`\s`)
func TestVersion14Hello(t *testing.T) {
// Tests that we can send and receive a version 0.14 hello message.
expected := Hello{
DeviceName: "test device",
ClientName: "syncthing",
ClientVersion: "v0.14.5",
}
msgBuf, err := expected.Marshal()
if err != nil {
t.Fatal(err)
}
hdrBuf := make([]byte, 8)
binary.BigEndian.PutUint32(hdrBuf, HelloMessageMagic)
binary.BigEndian.PutUint32(hdrBuf[4:], uint32(len(msgBuf)))
outBuf := new(bytes.Buffer)
outBuf.Write(hdrBuf)
outBuf.Write(msgBuf)
inBuf := new(bytes.Buffer)
conn := &readWriter{outBuf, inBuf}
send := &Hello{
DeviceName: "this device",
ClientName: "other client",
ClientVersion: "v0.14.6",
}
res, err := ExchangeHello(conn, send)
if err != nil {
t.Fatal(err)
}
if res.ClientName != expected.ClientName {
t.Errorf("incorrect ClientName %q != expected %q", res.ClientName, expected.ClientName)
}
if res.ClientVersion != expected.ClientVersion {
t.Errorf("incorrect ClientVersion %q != expected %q", res.ClientVersion, expected.ClientVersion)
}
if res.DeviceName != expected.DeviceName {
t.Errorf("incorrect DeviceName %q != expected %q", res.DeviceName, expected.DeviceName)
}
}
func TestVersion13Hello(t *testing.T) {
// Tests that we can send and receive a version 0.13 hello message.
@@ -42,8 +89,8 @@ func TestVersion13Hello(t *testing.T) {
}
res, err := ExchangeHello(conn, send)
if err != nil {
t.Fatal(err)
if err != ErrTooOldVersion13 {
t.Errorf("unexpected error %v != ErrTooOldVersion13", err)
}
if res.ClientName != expected.ClientName {
@@ -94,7 +141,7 @@ func TestVersion12Hello(t *testing.T) {
_, err := ExchangeHello(conn, send)
if err != ErrTooOldVersion12 {
t.Errorf("unexpected error %v != ErrTooOld", err)
t.Errorf("unexpected error %v != ErrTooOldVersion12", err)
}
}

186
lib/protocol/message.go
View File

@@ -1,186 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o message_xdr.go message.go
package protocol
import (
"bytes"
"crypto/sha256"
"fmt"
)
var (
sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
)
type IndexMessage struct {
Folder string // max:256
Files []FileInfo // max:1000000
Flags uint32
Options []Option // max:64
}
type FileInfo struct {
Name string // max:8192
Flags uint32
Modified int64
Version Vector
LocalVersion int64
CachedSize int64 // noencode (cache only)
Blocks []BlockInfo // max:10000000
}
func (f FileInfo) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
}
func (f FileInfo) Size() (bytes int64) {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
if f.CachedSize > 0 {
return f.CachedSize
}
for _, b := range f.Blocks {
bytes += int64(b.Size)
}
f.CachedSize = bytes
return
}
func (f FileInfo) IsDeleted() bool {
return f.Flags&FlagDeleted != 0
}
func (f FileInfo) IsInvalid() bool {
return f.Flags&FlagInvalid != 0
}
func (f FileInfo) IsDirectory() bool {
return f.Flags&FlagDirectory != 0
}
func (f FileInfo) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfo) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {
// If a modification is in conflict with a delete, we pick the
// modification.
if !f.IsDeleted() && other.IsDeleted() {
return true
}
if f.IsDeleted() && !other.IsDeleted() {
return false
}
// The one with the newer modification time wins.
if f.Modified > other.Modified {
return true
}
if f.Modified < other.Modified {
return false
}
// The modification times were equal. Use the device ID in the version
// vector as tie breaker.
return f.Version.Compare(other.Version) == ConcurrentGreater
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size int32
Hash []byte // max:64
}
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
// IsEmpty returns true if the block is a full block of zeroes.
func (b BlockInfo) IsEmpty() bool {
return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
}
type RequestMessage struct {
Folder string // max:256
Name string // max:8192
Offset int64
Size int32
Hash []byte // max:64
Flags uint32
Options []Option // max:64
}
type ResponseMessage struct {
Data []byte
Code int32
}
type ClusterConfigMessage struct {
Folders []Folder // max:1000000
Options []Option // max:64
}
type DownloadProgressMessage struct {
Folder string // max:64
Updates []FileDownloadProgressUpdate // max:1000000
Flags uint32
Options []Option // max:64
}
func (o *ClusterConfigMessage) GetOption(key string) string {
for _, option := range o.Options {
if option.Key == key {
return option.Value
}
}
return ""
}
type Folder struct {
ID string // max:256
Label string // max:256
Devices []Device // max:1000000
Flags uint32
Options []Option // max:64
}
type Device struct {
ID []byte // max:32
Name string // max:64
Addresses []string // max:64,2083
Compression uint32
CertName string // max:64
MaxLocalVersion int64
Flags uint32
Options []Option // max:64
}
type FileDownloadProgressUpdate struct {
UpdateType uint32
Name string // max:8192
Version Vector
BlockIndexes []int32 // max:1000000
}
type Option struct {
Key string // max:64
Value string // max:1024
}
type CloseMessage struct {
Reason string // max:1024
Code int32
}
type EmptyMessage struct{}

1309
lib/protocol/message_xdr.go
View File

File diff suppressed because it is too large Load Diff

12
lib/protocol/nativemodel_darwin.go
View File

@@ -12,21 +12,21 @@ type nativeModel struct {
Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.Model.Index(deviceID, folder, files, flags, options)
m.Model.Index(deviceID, folder, files)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
for i := range files {
files[i].Name = norm.NFD.String(files[i].Name)
}
m.Model.IndexUpdate(deviceID, folder, files, flags, options)
m.Model.IndexUpdate(deviceID, folder, files)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
name = norm.NFD.String(name)
return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
}

14
lib/protocol/nativemodel_windows.go
View File

@@ -24,19 +24,19 @@ type nativeModel struct {
Model
}
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
fixupFiles(folder, files)
m.Model.Index(deviceID, folder, files, flags, options)
m.Model.Index(deviceID, folder, files)
}
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
fixupFiles(folder, files)
m.Model.IndexUpdate(deviceID, folder, files, flags, options)
m.Model.IndexUpdate(deviceID, folder, files)
}
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
name = filepath.FromSlash(name)
return m.Model.Request(deviceID, folder, name, offset, hash, flags, options, buf)
return m.Model.Request(deviceID, folder, name, offset, hash, fromTemporary, buf)
}
func fixupFiles(folder string, files []FileInfo) {
@@ -47,7 +47,7 @@ func fixupFiles(folder string, files []FileInfo) {
// can't possibly exist here anyway.
continue
}
files[i].Flags |= FlagInvalid
files[i].Invalid = true
l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder)
}
files[i].Name = filepath.FromSlash(files[i].Name)

723
lib/protocol/protocol.go
View File

File diff suppressed because it is too large Load Diff

353
lib/protocol/protocol_test.go
View File

@@ -3,21 +3,17 @@
package protocol
import (
"encoding/binary"
"encoding/hex"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"reflect"
"strings"
"testing"
"testing/quick"
"time"
"github.com/calmh/xdr"
"github.com/syncthing/syncthing/lib/rand"
)
var (
@@ -26,64 +22,6 @@ var (
quickCfg = &quick.Config{}
)
func TestHeaderEncodeDecode(t *testing.T) {
f := func(ver, id, typ int) bool {
ver = int(uint(ver) % 16)
id = int(uint(id) % 4096)
typ = int(uint(typ) % 256)
h0 := header{version: ver, msgID: id, msgType: typ}
h1 := decodeHeader(encodeHeader(h0))
return h0 == h1
}
if err := quick.Check(f, nil); err != nil {
t.Error(err)
}
}
func TestHeaderMarshalUnmarshal(t *testing.T) {
f := func(ver, id, typ int) bool {
ver = int(uint(ver) % 16)
id = int(uint(id) % 4096)
typ = int(uint(typ) % 256)
buf := make([]byte, 4)
h0 := header{version: ver, msgID: id, msgType: typ}
h0.MarshalXDRInto(&xdr.Marshaller{Data: buf})
var h1 header
h1.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: buf})
return h0 == h1
}
if err := quick.Check(f, nil); err != nil {
t.Error(err)
}
}
func TestHeaderLayout(t *testing.T) {
var e, a uint32
// Version are the first four bits
e = 0xf0000000
a = encodeHeader(header{version: 0xf})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
// Message ID are the following 12 bits
e = 0x0fff0000
a = encodeHeader(header{msgID: 0xfff})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
// Type are the last 8 bits before reserved
e = 0x0000ff00
a = encodeHeader(header{msgType: 0xff})
if a != e {
t.Errorf("Header layout incorrect; %08x != %08x", a, e)
}
}
func TestPing(t *testing.T) {
ar, aw := io.Pipe()
br, bw := io.Pipe()
@@ -92,8 +30,8 @@ func TestPing(t *testing.T) {
c0.Start()
c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
c1.Start()
c0.ClusterConfig(ClusterConfigMessage{})
c1.ClusterConfig(ClusterConfigMessage{})
c0.ClusterConfig(ClusterConfig{})
c1.ClusterConfig(ClusterConfig{})
if ok := c0.ping(); !ok {
t.Error("c0 ping failed")
@@ -103,56 +41,6 @@ func TestPing(t *testing.T) {
}
}
func TestVersionErr(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
c0.Start()
c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
c1.Start()
c0.ClusterConfig(ClusterConfigMessage{})
c1.ClusterConfig(ClusterConfigMessage{})
timeoutWriteHeader(c0.cw, header{
version: 2, // higher than supported
msgID: 0,
msgType: messageTypeIndex,
})
if err := m1.closedError(); err == nil || !strings.Contains(err.Error(), "unknown protocol version") {
t.Error("Connection should close due to unknown version, not", err)
}
}
func TestTypeErr(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).Connection.(*rawConnection)
c0.Start()
c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
c1.Start()
c0.ClusterConfig(ClusterConfigMessage{})
c1.ClusterConfig(ClusterConfigMessage{})
timeoutWriteHeader(c0.cw, header{
version: 0,
msgID: 0,
msgType: 42, // unknown type
})
if err := m1.closedError(); err == nil || !strings.Contains(err.Error(), "unknown message type") {
t.Error("Connection should close due to unknown message type, not", err)
}
}
func TestClose(t *testing.T) {
m0 := newTestModel()
m1 := newTestModel()
@@ -164,8 +52,8 @@ func TestClose(t *testing.T) {
c0.Start()
c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
c1.Start()
c0.ClusterConfig(ClusterConfigMessage{})
c1.ClusterConfig(ClusterConfigMessage{})
c0.ClusterConfig(ClusterConfig{})
c1.ClusterConfig(ClusterConfig{})
c0.close(errors.New("manual close"))
@@ -180,8 +68,8 @@ func TestClose(t *testing.T) {
t.Error("Ping should not return true")
}
c0.Index("default", nil, 0, nil)
c0.Index("default", nil, 0, nil)
c0.Index("default", nil)
c0.Index("default", nil)
if _, err := c0.Request("default", "foo", 0, 0, nil, false); err == nil {
t.Error("Request should return an error")
@@ -193,15 +81,11 @@ func TestMarshalIndexMessage(t *testing.T) {
quickCfg.MaxCount = 10
}
f := func(m1 IndexMessage) bool {
if len(m1.Options) == 0 {
m1.Options = nil
}
f := func(m1 Index) bool {
if len(m1.Files) == 0 {
m1.Files = nil
}
for i, f := range m1.Files {
m1.Files[i].CachedSize = 0
if len(f.Blocks) == 0 {
m1.Files[i].Blocks = nil
} else {
@@ -212,9 +96,12 @@ func TestMarshalIndexMessage(t *testing.T) {
}
}
}
if len(f.Version.Counters) == 0 {
m1.Files[i].Version.Counters = nil
}
}
return testMarshal(t, "index", &m1, &IndexMessage{})
return testMarshal(t, "index", &m1, &Index{})
}
if err := quick.Check(f, quickCfg); err != nil {
@@ -227,14 +114,11 @@ func TestMarshalRequestMessage(t *testing.T) {
quickCfg.MaxCount = 10
}
f := func(m1 RequestMessage) bool {
if len(m1.Options) == 0 {
m1.Options = nil
}
f := func(m1 Request) bool {
if len(m1.Hash) == 0 {
m1.Hash = nil
}
return testMarshal(t, "request", &m1, &RequestMessage{})
return testMarshal(t, "request", &m1, &Request{})
}
if err := quick.Check(f, quickCfg); err != nil {
@@ -247,11 +131,11 @@ func TestMarshalResponseMessage(t *testing.T) {
quickCfg.MaxCount = 10
}
f := func(m1 ResponseMessage) bool {
f := func(m1 Response) bool {
if len(m1.Data) == 0 {
m1.Data = nil
}
return testMarshal(t, "response", &m1, &ResponseMessage{})
return testMarshal(t, "response", &m1, &Response{})
}
if err := quick.Check(f, quickCfg); err != nil {
@@ -264,10 +148,7 @@ func TestMarshalClusterConfigMessage(t *testing.T) {
quickCfg.MaxCount = 10
}
f := func(m1 ClusterConfigMessage) bool {
if len(m1.Options) == 0 {
m1.Options = nil
}
f := func(m1 ClusterConfig) bool {
if len(m1.Folders) == 0 {
m1.Folders = nil
}
@@ -275,11 +156,8 @@ func TestMarshalClusterConfigMessage(t *testing.T) {
if len(m1.Folders[i].Devices) == 0 {
m1.Folders[i].Devices = nil
}
if len(m1.Folders[i].Options) == 0 {
m1.Folders[i].Options = nil
}
}
return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
return testMarshal(t, "clusterconfig", &m1, &ClusterConfig{})
}
if err := quick.Check(f, quickCfg); err != nil {
@@ -292,8 +170,8 @@ func TestMarshalCloseMessage(t *testing.T) {
quickCfg.MaxCount = 10
}
f := func(m1 CloseMessage) bool {
return testMarshal(t, "close", &m1, &CloseMessage{})
f := func(m1 Close) bool {
return testMarshal(t, "close", &m1, &Close{})
}
if err := quick.Check(f, quickCfg); err != nil {
@@ -301,108 +179,97 @@ func TestMarshalCloseMessage(t *testing.T) {
}
}
type message interface {
MarshalXDR() ([]byte, error)
UnmarshalXDR([]byte) error
}
func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
failed := func(bc []byte) {
bs, _ := json.MarshalIndent(m1, "", " ")
ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
bs, _ = json.MarshalIndent(m2, "", " ")
ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
if len(bc) > 0 {
f, _ := os.Create(prefix + "-data.txt")
fmt.Fprint(f, hex.Dump(bc))
f.Close()
buf, err := m1.Marshal()
if err != nil {
t.Fatal(err)
}
err = m2.Unmarshal(buf)
if err != nil {
t.Fatal(err)
}
bs1, _ := json.MarshalIndent(m1, "", " ")
bs2, _ := json.MarshalIndent(m2, "", " ")
if !bytes.Equal(bs1, bs2) {
ioutil.WriteFile(prefix+"-1.txt", bs1, 0644)
ioutil.WriteFile(prefix+"-2.txt", bs1, 0644)
return false
}
return true
}
func TestMarshalledIndexMessageSize(t *testing.T) {
// We should be able to handle a 1 TiB file without
// blowing the default max message size.
if testing.Short() {
t.Skip("this test requires a lot of memory")
return
}
const (
maxMessageSize = MaxMessageLen
fileSize = 1 << 40
blockSize = BlockSize
)
f := FileInfo{
Name: "a normal length file name withoout any weird stuff.txt",
Type: FileInfoTypeFile,
Size: fileSize,
Permissions: 0666,
Modified: time.Now().Unix(),
Version: Vector{Counters: []Counter{{ID: 1 << 60, Value: 1}, {ID: 2 << 60, Value: 1}}},
Blocks: make([]BlockInfo, fileSize/blockSize),
}
for i := 0; i < fileSize/blockSize; i++ {
f.Blocks[i].Offset = int64(i) * blockSize
f.Blocks[i].Size = blockSize
f.Blocks[i].Hash = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 1, 2}
}
idx := Index{
Folder: "some folder ID",
Files: []FileInfo{f},
}
msgSize := idx.ProtoSize()
if msgSize > maxMessageSize {
t.Errorf("Message size %d bytes is larger than max %d", msgSize, maxMessageSize)
}
}
func TestLZ4Compression(t *testing.T) {
c := new(rawConnection)
for i := 0; i < 10; i++ {
dataLen := 150 + rand.Intn(150)
data := make([]byte, dataLen)
_, err := io.ReadFull(rand.Reader, data[100:])
if err != nil {
t.Fatal(err)
}
comp, err := c.lz4Compress(data)
if err != nil {
t.Errorf("compressing %d bytes: %v", dataLen, err)
continue
}
}
buf, err := m1.MarshalXDR()
if err != nil && strings.Contains(err.Error(), "exceeds size") {
return true
}
if err != nil {
failed(nil)
t.Fatal(err)
}
err = m2.UnmarshalXDR(buf)
if err != nil {
failed(buf)
t.Fatal(err)
}
ok := reflect.DeepEqual(m1, m2)
if !ok {
failed(buf)
}
return ok
}
func timeoutWriteHeader(w io.Writer, hdr header) {
// This tries to write a message header to w, but times out after a while.
// This is useful because in testing, with a PipeWriter, it will block
// forever if the other side isn't reading any more. On the other hand we
// can't just "go" it into the background, because if the other side is
// still there we should wait for the write to complete. Yay.
var buf [8]byte // header and message length
binary.BigEndian.PutUint32(buf[:], encodeHeader(hdr))
binary.BigEndian.PutUint32(buf[4:], 0) // zero message length, explicitly
done := make(chan struct{})
go func() {
w.Write(buf[:])
close(done)
}()
select {
case <-done:
case <-time.After(250 * time.Millisecond):
}
}
func TestFileInfoSize(t *testing.T) {
fi := FileInfo{
Blocks: []BlockInfo{
{Size: 42},
{Offset: 42, Size: 23},
{Offset: 42 + 23, Size: 34},
},
}
size := fi.Size()
want := int64(42 + 23 + 34)
if size != want {
t.Errorf("Incorrect size reported, got %d, want %d", size, want)
}
size = fi.Size() // Cached, this time
if size != want {
t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
}
fi.CachedSize = 8
want = 8
size = fi.Size() // Ensure it came from the cache
if size != want {
t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
}
fi.CachedSize = 0
fi.Flags = FlagDirectory
want = 128
size = fi.Size() // Directories are 128 bytes large
if size != want {
t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
}
fi.CachedSize = 0
fi.Flags = FlagDeleted
want = 128
size = fi.Size() // Also deleted files
if size != want {
t.Errorf("Incorrect cached size reported, got %d, want %d", size, want)
res, err := c.lz4Decompress(comp)
if err != nil {
t.Errorf("decompressing %d bytes to %d: %v", len(comp), dataLen, err)
continue
}
if len(res) != len(data) {
t.Errorf("Incorrect len %d != expected %d", len(res), len(data))
}
if !bytes.Equal(data, res) {
t.Error("Incorrect decompressed data")
}
t.Logf("OK #%d, %d -> %d -> %d", i, dataLen, len(comp), dataLen)
}
}

143
lib/protocol/vector.go
View File

@@ -6,35 +6,30 @@ package protocol
// version vector. The vector has slice semantics and some operations on it
// are "append-like" in that they may return the same vector modified, or v
// new allocated Vector with the modified contents.
type Vector []Counter
// Counter represents a single counter in the version vector.
type Counter struct {
ID ShortID
Value uint64
}
// Update returns a Vector with the index for the specific ID incremented by
// one. If it is possible, the vector v is updated and returned. If it is not,
// a copy will be created, updated and returned.
func (v Vector) Update(id ShortID) Vector {
for i := range v {
if v[i].ID == id {
for i := range v.Counters {
if v.Counters[i].ID == id {
// Update an existing index
v[i].Value++
v.Counters[i].Value++
return v
} else if v[i].ID > id {
} else if v.Counters[i].ID > id {
// Insert a new index
nv := make(Vector, len(v)+1)
copy(nv, v[:i])
nv := make([]Counter, len(v.Counters)+1)
copy(nv, v.Counters[:i])
nv[i].ID = id
nv[i].Value = 1
copy(nv[i+1:], v[i:])
return nv
copy(nv[i+1:], v.Counters[i:])
return Vector{nv}
}
}
// Append a new index
return append(v, Counter{id, 1})
return Vector{append(v.Counters, Counter{id, 1})}
}
// Merge returns the vector containing the maximum indexes from v and b. If it
@@ -42,28 +37,28 @@ func (v Vector) Update(id ShortID) Vector {
// will be created, updated and returned.
func (v Vector) Merge(b Vector) Vector {
var vi, bi int
for bi < len(b) {
if vi == len(v) {
for bi < len(b.Counters) {
if vi == len(v.Counters) {
// We've reach the end of v, all that remains are appends
return append(v, b[bi:]...)
return Vector{append(v.Counters, b.Counters[bi:]...)}
}
if v[vi].ID > b[bi].ID {
if v.Counters[vi].ID > b.Counters[bi].ID {
// The index from b should be inserted here
n := make(Vector, len(v)+1)
copy(n, v[:vi])
n[vi] = b[bi]
copy(n[vi+1:], v[vi:])
v = n
n := make([]Counter, len(v.Counters)+1)
copy(n, v.Counters[:vi])
n[vi] = b.Counters[bi]
copy(n[vi+1:], v.Counters[vi:])
v.Counters = n
}
if v[vi].ID == b[bi].ID {
if val := b[bi].Value; val > v[vi].Value {
v[vi].Value = val
if v.Counters[vi].ID == b.Counters[bi].ID {
if val := b.Counters[bi].Value; val > v.Counters[vi].Value {
v.Counters[vi].Value = val
}
}
if bi < len(b) && v[vi].ID == b[bi].ID {
if bi < len(b.Counters) && v.Counters[vi].ID == b.Counters[bi].ID {
bi++
}
vi++
@@ -74,9 +69,9 @@ func (v Vector) Merge(b Vector) Vector {
// Copy returns an identical vector that is not shared with v.
func (v Vector) Copy() Vector {
nv := make(Vector, len(v))
copy(nv, v)
return nv
nv := make([]Counter, len(v.Counters))
copy(nv, v.Counters)
return Vector{nv}
}
// Equal returns true when the two vectors are equivalent.
@@ -106,10 +101,96 @@ func (v Vector) Concurrent(b Vector) bool {
// Counter returns the current value of the given counter ID.
func (v Vector) Counter(id ShortID) uint64 {
for _, c := range v {
for _, c := range v.Counters {
if c.ID == id {
return c.Value
}
}
return 0
}
// Ordering represents the relationship between two Vectors.
type Ordering int
const (
Equal Ordering = iota
Greater
Lesser
ConcurrentLesser
ConcurrentGreater
)
// There's really no such thing as "concurrent lesser" and "concurrent
// greater" in version vectors, just "concurrent". But it's useful to be able
// to get a strict ordering between versions for stable sorts and so on, so we
// return both variants. The convenience method Concurrent() can be used to
// check for either case.
// Compare returns the Ordering that describes a's relation to b.
func (v Vector) Compare(b Vector) Ordering {
var ai, bi int // index into a and b
var av, bv Counter // value at current index
result := Equal
for ai < len(v.Counters) || bi < len(b.Counters) {
var aMissing, bMissing bool
if ai < len(v.Counters) {
av = v.Counters[ai]
} else {
av = Counter{}
aMissing = true
}
if bi < len(b.Counters) {
bv = b.Counters[bi]
} else {
bv = Counter{}
bMissing = true
}
switch {
case av.ID == bv.ID:
// We have a counter value for each side
if av.Value > bv.Value {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
} else if av.Value < bv.Value {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
case !aMissing && av.ID < bv.ID || bMissing:
// Value is missing on the b side
if av.Value > 0 {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
}
case !bMissing && bv.ID < av.ID || aMissing:
// Value is missing on the a side
if bv.Value > 0 {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
}
if ai < len(v.Counters) && (av.ID <= bv.ID || bMissing) {
ai++
}
if bi < len(b.Counters) && (bv.ID <= av.ID || aMissing) {
bi++
}
}
return result
}

89
lib/protocol/vector_compare.go
View File

@@ -1,89 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
// Ordering represents the relationship between two Vectors.
type Ordering int
const (
Equal Ordering = iota
Greater
Lesser
ConcurrentLesser
ConcurrentGreater
)
// There's really no such thing as "concurrent lesser" and "concurrent
// greater" in version vectors, just "concurrent". But it's useful to be able
// to get a strict ordering between versions for stable sorts and so on, so we
// return both variants. The convenience method Concurrent() can be used to
// check for either case.
// Compare returns the Ordering that describes a's relation to b.
func (a Vector) Compare(b Vector) Ordering {
var ai, bi int // index into a and b
var av, bv Counter // value at current index
result := Equal
for ai < len(a) || bi < len(b) {
var aMissing, bMissing bool
if ai < len(a) {
av = a[ai]
} else {
av = Counter{}
aMissing = true
}
if bi < len(b) {
bv = b[bi]
} else {
bv = Counter{}
bMissing = true
}
switch {
case av.ID == bv.ID:
// We have a counter value for each side
if av.Value > bv.Value {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
} else if av.Value < bv.Value {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
case !aMissing && av.ID < bv.ID || bMissing:
// Value is missing on the b side
if av.Value > 0 {
if result == Lesser {
return ConcurrentLesser
}
result = Greater
}
case !bMissing && bv.ID < av.ID || aMissing:
// Value is missing on the a side
if bv.Value > 0 {
if result == Greater {
return ConcurrentGreater
}
result = Lesser
}
}
if ai < len(a) && (av.ID <= bv.ID || bMissing) {
ai++
}
if bi < len(b) && (bv.ID <= av.ID || aMissing) {
bi++
}
}
return result
}

249
lib/protocol/vector_compare_test.go
View File

@@ -1,249 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import (
"math"
"testing"
)
func TestCompare(t *testing.T) {
testcases := []struct {
a, b Vector
r Ordering
}{
// Empty vectors are identical
{Vector{}, Vector{}, Equal},
{Vector{}, nil, Equal},
{nil, Vector{}, Equal},
{nil, Vector{Counter{42, 0}}, Equal},
{Vector{}, Vector{Counter{42, 0}}, Equal},
{Vector{Counter{42, 0}}, nil, Equal},
{Vector{Counter{42, 0}}, Vector{}, Equal},
// Zero is the implied value for a missing Counter
{
Vector{Counter{42, 0}},
Vector{Counter{77, 0}},
Equal,
},
// Equal vectors are equal
{
Vector{Counter{42, 33}},
Vector{Counter{42, 33}},
Equal,
},
{
Vector{Counter{42, 33}, Counter{77, 24}},
Vector{Counter{42, 33}, Counter{77, 24}},
Equal,
},
// These a-vectors are all greater than the b-vector
{
Vector{Counter{42, 1}},
nil,
Greater,
},
{
Vector{Counter{42, 1}},
Vector{},
Greater,
},
{
Vector{Counter{0, 1}},
Vector{Counter{0, 0}},
Greater,
},
{
Vector{Counter{42, 1}},
Vector{Counter{42, 0}},
Greater,
},
{
Vector{Counter{math.MaxUint64, 1}},
Vector{Counter{math.MaxUint64, 0}},
Greater,
},
{
Vector{Counter{0, math.MaxUint64}},
Vector{Counter{0, 0}},
Greater,
},
{
Vector{Counter{42, math.MaxUint64}},
Vector{Counter{42, 0}},
Greater,
},
{
Vector{Counter{math.MaxUint64, math.MaxUint64}},
Vector{Counter{math.MaxUint64, 0}},
Greater,
},
{
Vector{Counter{0, math.MaxUint64}},
Vector{Counter{0, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{42, math.MaxUint64}},
Vector{Counter{42, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{math.MaxUint64, math.MaxUint64}},
Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}},
Greater,
},
{
Vector{Counter{42, 2}},
Vector{Counter{42, 1}},
Greater,
},
{
Vector{Counter{22, 22}, Counter{42, 2}},
Vector{Counter{22, 22}, Counter{42, 1}},
Greater,
},
{
Vector{Counter{42, 2}, Counter{77, 3}},
Vector{Counter{42, 1}, Counter{77, 3}},
Greater,
},
{
Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Greater,
},
{
Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Greater,
},
// These a-vectors are all lesser than the b-vector
{nil, Vector{Counter{42, 1}}, Lesser},
{Vector{}, Vector{Counter{42, 1}}, Lesser},
{
Vector{Counter{42, 0}},
Vector{Counter{42, 1}},
Lesser,
},
{
Vector{Counter{42, 1}},
Vector{Counter{42, 2}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}},
Vector{Counter{22, 22}, Counter{42, 2}},
Lesser,
},
{
Vector{Counter{42, 1}, Counter{77, 3}},
Vector{Counter{42, 2}, Counter{77, 3}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
Lesser,
},
{
Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
Lesser,
},
// These are all in conflict
{
Vector{Counter{42, 2}},
Vector{Counter{43, 1}},
ConcurrentGreater,
},
{
Vector{Counter{43, 1}},
Vector{Counter{42, 2}},
ConcurrentLesser,
},
{
Vector{Counter{22, 23}, Counter{42, 1}},
Vector{Counter{22, 22}, Counter{42, 2}},
ConcurrentGreater,
},
{
Vector{Counter{22, 21}, Counter{42, 2}},
Vector{Counter{22, 22}, Counter{42, 1}},
ConcurrentLesser,
},
{
Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}},
Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}},
ConcurrentLesser,
},
}
for i, tc := range testcases {
// Test real Compare
if r := tc.a.Compare(tc.b); r != tc.r {
t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
}
// Test convenience functions
switch tc.r {
case Greater:
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
case Lesser:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case Equal:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.Equal(tc.b) {
t.Errorf("%+v not == %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case ConcurrentLesser, ConcurrentGreater:
if !tc.a.Concurrent(tc.b) {
t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
}
}
}

294
lib/protocol/vector_test.go
View File

@@ -2,7 +2,10 @@
package protocol
import "testing"
import (
"math"
"testing"
)
func TestUpdate(t *testing.T) {
var v Vector
@@ -10,7 +13,7 @@ func TestUpdate(t *testing.T) {
// Append
v = v.Update(42)
expected := Vector{Counter{42, 1}}
expected := Vector{[]Counter{{42, 1}}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
@@ -19,7 +22,7 @@ func TestUpdate(t *testing.T) {
// Insert at front
v = v.Update(36)
expected = Vector{Counter{36, 1}, Counter{42, 1}}
expected = Vector{[]Counter{{36, 1}, {42, 1}}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
@@ -28,7 +31,7 @@ func TestUpdate(t *testing.T) {
// Insert in moddle
v = v.Update(37)
expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}}
expected = Vector{[]Counter{{36, 1}, {37, 1}, {42, 1}}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
@@ -37,7 +40,7 @@ func TestUpdate(t *testing.T) {
// Update existing
v = v.Update(37)
expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}}
expected = Vector{[]Counter{{36, 1}, {37, 2}, {42, 1}}}
if v.Compare(expected) != Equal {
t.Errorf("Update error, %+v != %+v", v, expected)
@@ -45,7 +48,7 @@ func TestUpdate(t *testing.T) {
}
func TestCopy(t *testing.T) {
v0 := Vector{Counter{42, 1}}
v0 := Vector{[]Counter{{42, 1}}}
v1 := v0.Copy()
v1.Update(42)
if v0.Compare(v1) != Lesser {
@@ -64,52 +67,52 @@ func TestMerge(t *testing.T) {
Vector{},
},
{
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{[]Counter{{22, 1}, {42, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
},
// Appends
{
Vector{},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{[]Counter{{22, 1}, {42, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
},
{
Vector{Counter{22, 1}},
Vector{Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{[]Counter{{22, 1}}},
Vector{[]Counter{{42, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
},
{
Vector{Counter{22, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{[]Counter{{22, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
},
// Insert
{
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
Vector{[]Counter{{22, 1}, {42, 1}}},
Vector{[]Counter{{22, 1}, {23, 2}, {42, 1}}},
Vector{[]Counter{{22, 1}, {23, 2}, {42, 1}}},
},
{
Vector{Counter{42, 1}},
Vector{Counter{22, 1}},
Vector{Counter{22, 1}, Counter{42, 1}},
Vector{[]Counter{{42, 1}}},
Vector{[]Counter{{22, 1}}},
Vector{[]Counter{{22, 1}, {42, 1}}},
},
// Update
{
Vector{Counter{22, 1}, Counter{42, 2}},
Vector{Counter{22, 2}, Counter{42, 1}},
Vector{Counter{22, 2}, Counter{42, 2}},
Vector{[]Counter{{22, 1}, {42, 2}}},
Vector{[]Counter{{22, 2}, {42, 1}}},
Vector{[]Counter{{22, 2}, {42, 2}}},
},
// All of the above
{
Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}},
Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}},
Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}},
Vector{[]Counter{{10, 1}, {20, 2}, {30, 1}}},
Vector{[]Counter{{5, 1}, {10, 2}, {15, 1}, {20, 1}, {25, 1}, {35, 1}}},
Vector{[]Counter{{5, 1}, {10, 2}, {15, 1}, {20, 2}, {25, 1}, {30, 1}, {35, 1}}},
},
}
@@ -121,7 +124,7 @@ func TestMerge(t *testing.T) {
}
func TestCounterValue(t *testing.T) {
v0 := Vector{Counter{42, 1}, Counter{64, 5}}
v0 := Vector{[]Counter{{42, 1}, {64, 5}}}
if v0.Counter(42) != 1 {
t.Errorf("Counter error, %d != %d", v0.Counter(42), 1)
}
@@ -132,3 +135,234 @@ func TestCounterValue(t *testing.T) {
t.Errorf("Counter error, %d != %d", v0.Counter(72), 0)
}
}
func TestCompare(t *testing.T) {
testcases := []struct {
a, b Vector
r Ordering
}{
// Empty vectors are identical
{Vector{}, Vector{}, Equal},
{Vector{}, Vector{[]Counter{{42, 0}}}, Equal},
{Vector{[]Counter{Counter{42, 0}}}, Vector{}, Equal},
// Zero is the implied value for a missing Counter
{
Vector{[]Counter{{42, 0}}},
Vector{[]Counter{{77, 0}}},
Equal,
},
// Equal vectors are equal
{
Vector{[]Counter{{42, 33}}},
Vector{[]Counter{{42, 33}}},
Equal,
},
{
Vector{[]Counter{{42, 33}, {77, 24}}},
Vector{[]Counter{{42, 33}, {77, 24}}},
Equal,
},
// These a-vectors are all greater than the b-vector
{
Vector{[]Counter{{42, 1}}},
Vector{},
Greater,
},
{
Vector{[]Counter{{0, 1}}},
Vector{[]Counter{{0, 0}}},
Greater,
},
{
Vector{[]Counter{{42, 1}}},
Vector{[]Counter{{42, 0}}},
Greater,
},
{
Vector{[]Counter{{math.MaxUint64, 1}}},
Vector{[]Counter{{math.MaxUint64, 0}}},
Greater,
},
{
Vector{[]Counter{{0, math.MaxUint64}}},
Vector{[]Counter{{0, 0}}},
Greater,
},
{
Vector{[]Counter{{42, math.MaxUint64}}},
Vector{[]Counter{{42, 0}}},
Greater,
},
{
Vector{[]Counter{{math.MaxUint64, math.MaxUint64}}},
Vector{[]Counter{{math.MaxUint64, 0}}},
Greater,
},
{
Vector{[]Counter{{0, math.MaxUint64}}},
Vector{[]Counter{{0, math.MaxUint64 - 1}}},
Greater,
},
{
Vector{[]Counter{{42, math.MaxUint64}}},
Vector{[]Counter{{42, math.MaxUint64 - 1}}},
Greater,
},
{
Vector{[]Counter{{math.MaxUint64, math.MaxUint64}}},
Vector{[]Counter{{math.MaxUint64, math.MaxUint64 - 1}}},
Greater,
},
{
Vector{[]Counter{{42, 2}}},
Vector{[]Counter{{42, 1}}},
Greater,
},
{
Vector{[]Counter{{22, 22}, {42, 2}}},
Vector{[]Counter{{22, 22}, {42, 1}}},
Greater,
},
{
Vector{[]Counter{{42, 2}, {77, 3}}},
Vector{[]Counter{{42, 1}, {77, 3}}},
Greater,
},
{
Vector{[]Counter{{22, 22}, {42, 2}, {77, 3}}},
Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
Greater,
},
{
Vector{[]Counter{{22, 23}, {42, 2}, {77, 4}}},
Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
Greater,
},
// These a-vectors are all lesser than the b-vector
{Vector{}, Vector{[]Counter{{42, 1}}}, Lesser},
{
Vector{[]Counter{{42, 0}}},
Vector{[]Counter{{42, 1}}},
Lesser,
},
{
Vector{[]Counter{{42, 1}}},
Vector{[]Counter{{42, 2}}},
Lesser,
},
{
Vector{[]Counter{{22, 22}, {42, 1}}},
Vector{[]Counter{{22, 22}, {42, 2}}},
Lesser,
},
{
Vector{[]Counter{{42, 1}, {77, 3}}},
Vector{[]Counter{{42, 2}, {77, 3}}},
Lesser,
},
{
Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
Vector{[]Counter{{22, 22}, {42, 2}, {77, 3}}},
Lesser,
},
{
Vector{[]Counter{{22, 22}, {42, 1}, {77, 3}}},
Vector{[]Counter{{22, 23}, {42, 2}, {77, 4}}},
Lesser,
},
// These are all in conflict
{
Vector{[]Counter{{42, 2}}},
Vector{[]Counter{{43, 1}}},
ConcurrentGreater,
},
{
Vector{[]Counter{{43, 1}}},
Vector{[]Counter{{42, 2}}},
ConcurrentLesser,
},
{
Vector{[]Counter{{22, 23}, {42, 1}}},
Vector{[]Counter{{22, 22}, {42, 2}}},
ConcurrentGreater,
},
{
Vector{[]Counter{{22, 21}, {42, 2}}},
Vector{[]Counter{{22, 22}, {42, 1}}},
ConcurrentLesser,
},
{
Vector{[]Counter{{22, 21}, {42, 2}, {43, 1}}},
Vector{[]Counter{{20, 1}, {22, 22}, {42, 1}}},
ConcurrentLesser,
},
}
for i, tc := range testcases {
// Test real Compare
if r := tc.a.Compare(tc.b); r != tc.r {
t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
}
// Test convenience functions
switch tc.r {
case Greater:
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
case Lesser:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case Equal:
if tc.a.Concurrent(tc.b) {
t.Errorf("%+v concurrent %+v", tc.a, tc.b)
}
if !tc.a.Equal(tc.b) {
t.Errorf("%+v not == %+v", tc.a, tc.b)
}
if !tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
if !tc.a.LesserEqual(tc.b) {
t.Errorf("%+v not <= %+v", tc.a, tc.b)
}
case ConcurrentLesser, ConcurrentGreater:
if !tc.a.Concurrent(tc.b) {
t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
}
if tc.a.Equal(tc.b) {
t.Errorf("%+v == %+v", tc.a, tc.b)
}
if tc.a.GreaterEqual(tc.b) {
t.Errorf("%+v >= %+v", tc.a, tc.b)
}
if tc.a.LesserEqual(tc.b) {
t.Errorf("%+v <= %+v", tc.a, tc.b)
}
}
}
}

35
lib/protocol/vector_xdr.go
View File

@@ -1,35 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import "github.com/calmh/xdr"
// This stuff is hacked up manually because genxdr doesn't support 'type
// Vector []Counter' declarations and it was tricky when I tried to add it...
func (v Vector) MarshalXDRInto(m *xdr.Marshaller) error {
m.MarshalUint32(uint32(len(v)))
for i := range v {
m.MarshalUint64(uint64(v[i].ID))
m.MarshalUint64(v[i].Value)
}
return m.Error
}
func (v *Vector) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
l := int(u.UnmarshalUint32())
if l > 1e6 {
return xdr.ElementSizeExceeded("number of counters", l, 1e6)
}
n := make(Vector, l)
for i := range n {
n[i].ID = ShortID(u.UnmarshalUint64())
n[i].Value = u.UnmarshalUint64()
}
*v = n
return u.Error
}
func (v Vector) XDRSize() int {
return 4 + 16*len(v)
}

8
lib/protocol/wireformat.go
View File

@@ -12,7 +12,7 @@ type wireFormatConnection struct {
Connection
}
func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error {
func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
@@ -20,10 +20,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32,
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
return c.Connection.Index(folder, myFs, flags, options)
return c.Connection.Index(folder, myFs)
}
func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error {
func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
var myFs = make([]FileInfo, len(fs))
copy(myFs, fs)
@@ -31,7 +31,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags ui
myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
}
return c.Connection.IndexUpdate(folder, myFs, flags, options)
return c.Connection.IndexUpdate(folder, myFs)
}
func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {

2
lib/scanner/blockqueue.go
View File

@@ -71,7 +71,7 @@ func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo,
panic("Bug. Asked to hash a directory or a deleted file.")
}
blocks, err := HashFile(filepath.Join(dir, f.Name), blockSize, f.CachedSize, counter)
blocks, err := HashFile(filepath.Join(dir, f.Name), blockSize, f.Size, counter)
if err != nil {
l.Debugln("hash error:", f.Name, err)
continue

64
lib/scanner/walk.go
View File

@@ -168,7 +168,7 @@ func (w *walker) walk() (chan protocol.FileInfo, error) {
for file := range toHashChan {
filesToHash = append(filesToHash, file)
total += int64(file.CachedSize)
total += int64(file.Size)
}
realToHashChan := make(chan protocol.FileInfo)
@@ -309,25 +309,22 @@ func (w *walker) walkRegular(relPath string, info os.FileInfo, mtime time.Time,
// - was not invalid (since it looks valid now)
// - has the same size as previously
cf, ok := w.CurrentFiler.CurrentFile(relPath)
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, curMode)
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Permissions, curMode)
if ok && permUnchanged && !cf.IsDeleted() && cf.Modified == mtime.Unix() && !cf.IsDirectory() &&
!cf.IsSymlink() && !cf.IsInvalid() && cf.Size() == info.Size() {
!cf.IsSymlink() && !cf.IsInvalid() && cf.Size == info.Size() {
return nil
}
l.Debugln("rescan:", cf, mtime.Unix(), info.Mode()&os.ModePerm)
var flags = curMode & uint32(maskModePerm)
if w.IgnorePerms {
flags = protocol.FlagNoPermBits | 0666
}
f := protocol.FileInfo{
Name: relPath,
Version: cf.Version.Update(w.ShortID),
Flags: flags,
Modified: mtime.Unix(),
CachedSize: info.Size(),
Name: relPath,
Type: protocol.FileInfoTypeFile,
Version: cf.Version.Update(w.ShortID),
Permissions: curMode & uint32(maskModePerm),
NoPermissions: w.IgnorePerms,
Modified: mtime.Unix(),
Size: info.Size(),
}
l.Debugln("to hash:", relPath, f)
@@ -349,22 +346,18 @@ func (w *walker) walkDir(relPath string, info os.FileInfo, mtime time.Time, dcha
// - was not a symlink (since it's a directory now)
// - was not invalid (since it looks valid now)
cf, ok := w.CurrentFiler.CurrentFile(relPath)
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Permissions, uint32(info.Mode()))
if ok && permUnchanged && !cf.IsDeleted() && cf.IsDirectory() && !cf.IsSymlink() && !cf.IsInvalid() {
return nil
}
flags := uint32(protocol.FlagDirectory)
if w.IgnorePerms {
flags |= protocol.FlagNoPermBits | 0777
} else {
flags |= uint32(info.Mode() & maskModePerm)
}
f := protocol.FileInfo{
Name: relPath,
Version: cf.Version.Update(w.ShortID),
Flags: flags,
Modified: mtime.Unix(),
Name: relPath,
Type: protocol.FileInfoTypeDirectory,
Version: cf.Version.Update(w.ShortID),
Permissions: uint32(info.Mode() & maskModePerm),
NoPermissions: w.IgnorePerms,
Modified: mtime.Unix(),
}
l.Debugln("dir:", relPath, f)
@@ -420,11 +413,12 @@ func (w *walker) walkSymlink(absPath, relPath string, dchan chan protocol.FileIn
}
f := protocol.FileInfo{
Name: relPath,
Version: cf.Version.Update(w.ShortID),
Flags: uint32(protocol.FlagSymlink | protocol.FlagNoPermBits | 0666 | SymlinkFlags(targetType)),
Modified: 0,
Blocks: blocks,
Name: relPath,
Type: SymlinkType(targetType),
Version: cf.Version.Update(w.ShortID),
Modified: 0,
NoPermissions: true, // Symlinks don't have permissions of their own
Blocks: blocks,
}
l.Debugln("symlink changedb:", absPath, f)
@@ -519,21 +513,21 @@ func SymlinkTypeEqual(disk symlinks.TargetType, f protocol.FileInfo) bool {
case symlinks.TargetUnknown:
return true
case symlinks.TargetDirectory:
return f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0
return f.Type == protocol.FileInfoTypeSymlinkDirectory
case symlinks.TargetFile:
return !f.IsDirectory() && f.Flags&protocol.FlagSymlinkMissingTarget == 0
return f.Type == protocol.FileInfoTypeSymlinkFile
}
panic("unknown symlink TargetType")
}
func SymlinkFlags(t symlinks.TargetType) uint32 {
func SymlinkType(t symlinks.TargetType) protocol.FileInfoType {
switch t {
case symlinks.TargetFile:
return 0
return protocol.FileInfoTypeSymlinkFile
case symlinks.TargetDirectory:
return protocol.FlagDirectory
return protocol.FileInfoTypeSymlinkDirectory
case symlinks.TargetUnknown:
return protocol.FlagSymlinkMissingTarget
return protocol.FileInfoTypeSymlinkUnknown
}
panic("unknown symlink TargetType")
}

38
lib/scanner/walk_test.go
View File

@@ -28,9 +28,9 @@ import (
)
type testfile struct {
name string
size int
hash string
name string
length int64
hash string
}
type testfileList []testfile
@@ -322,7 +322,7 @@ func (l fileList) testfiles() testfileList {
if len(f.Blocks) > 1 {
panic("simple test case stuff only supports a single block per file")
}
testfiles[i] = testfile{name: f.Name, size: int(f.Size())}
testfiles[i] = testfile{name: f.Name, length: f.FileSize()}
if len(f.Blocks) == 1 {
testfiles[i].hash = fmt.Sprintf("%x", f.Blocks[0].Hash)
}
@@ -334,7 +334,7 @@ func (l testfileList) String() string {
var b bytes.Buffer
b.WriteString("{\n")
for _, f := range l {
fmt.Fprintf(&b, " %s (%d bytes): %s\n", f.name, f.size, f.hash)
fmt.Fprintf(&b, " %s (%d bytes): %s\n", f.name, f.length, f.hash)
}
b.WriteString("}")
return b.String()
@@ -342,28 +342,28 @@ func (l testfileList) String() string {
func TestSymlinkTypeEqual(t *testing.T) {
testcases := []struct {
onDiskType symlinks.TargetType
inIndexFlags uint32
equal bool
onDiskType symlinks.TargetType
fiType protocol.FileInfoType
equal bool
}{
// File is only equal to file
{symlinks.TargetFile, 0, true},
{symlinks.TargetFile, protocol.FlagDirectory, false},
{symlinks.TargetFile, protocol.FlagSymlinkMissingTarget, false},
{symlinks.TargetFile, protocol.FileInfoTypeSymlinkFile, true},
{symlinks.TargetFile, protocol.FileInfoTypeSymlinkDirectory, false},
{symlinks.TargetFile, protocol.FileInfoTypeSymlinkUnknown, false},
// Directory is only equal to directory
{symlinks.TargetDirectory, 0, false},
{symlinks.TargetDirectory, protocol.FlagDirectory, true},
{symlinks.TargetDirectory, protocol.FlagSymlinkMissingTarget, false},
{symlinks.TargetDirectory, protocol.FileInfoTypeSymlinkFile, false},
{symlinks.TargetDirectory, protocol.FileInfoTypeSymlinkDirectory, true},
{symlinks.TargetDirectory, protocol.FileInfoTypeSymlinkUnknown, false},
// Unknown is equal to anything
{symlinks.TargetUnknown, 0, true},
{symlinks.TargetUnknown, protocol.FlagDirectory, true},
{symlinks.TargetUnknown, protocol.FlagSymlinkMissingTarget, true},
{symlinks.TargetUnknown, protocol.FileInfoTypeSymlinkFile, true},
{symlinks.TargetUnknown, protocol.FileInfoTypeSymlinkDirectory, true},
{symlinks.TargetUnknown, protocol.FileInfoTypeSymlinkUnknown, true},
}
for _, tc := range testcases {
res := SymlinkTypeEqual(tc.onDiskType, protocol.FileInfo{Flags: tc.inIndexFlags})
res := SymlinkTypeEqual(tc.onDiskType, protocol.FileInfo{Type: tc.fiType})
if res != tc.equal {
t.Errorf("Incorrect result %v for %v, %v", res, tc.onDiskType, tc.inIndexFlags)
t.Errorf("Incorrect result %v for %v, %v", res, tc.onDiskType, tc.fiType)
}
}
}

7
lib/symlinks/symlink_windows.go
View File

@@ -13,7 +13,6 @@ import (
"path/filepath"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"syscall"
"unicode/utf16"
@@ -55,7 +54,7 @@ func init() {
path := filepath.Join(base, "symlinktest")
defer os.Remove(path)
err := Create(path, base, protocol.FlagDirectory)
err := Create(path, base, TargetDirectory)
if err != nil {
return
}
@@ -65,8 +64,8 @@ func init() {
return
}
target, flags, err := Read(path)
if err != nil || osutil.NativeFilename(target) != base || flags&protocol.FlagDirectory == 0 {
target, tt, err := Read(path)
if err != nil || osutil.NativeFilename(target) != base || tt != TargetDirectory {
return
}
Supported = true