perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/protocol, lib/discover, lib/db: Use protocol buffer serialization (fixes #3080)

Browse Source 
This changes the BEP protocol to use protocol buffer serialization
instead of XDR, and therefore also the database format. The local
discovery protocol is also updated to be protocol buffer format.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3276
LGTM: AudriusButkevicius
This commit is contained in:
Jakob Borg
2016-07-04 10:40:29 +00:00
committed by Audrius Butkevicius
parent 21f5b16e47
commit fa0101bd60
269 changed files with 477296 additions and 4175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/db/blockmap_test.go
View File

@@ -73,7 +73,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
f3.Flags |= protocol.FlagDirectory
f3.Type = protocol.FileInfoTypeDirectory
err := m.Add([]protocol.FileInfo{f1, f2, f3})
if err != nil {
@@ -99,9 +99,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
return true
})
f3.Flags = f1.Flags
f1.Flags |= protocol.FlagDeleted
f2.Flags |= protocol.FlagInvalid
f3.Permissions = f1.Permissions
f3.Deleted = f1.Deleted
f3.Invalid = f1.Invalid
f1.Deleted = true
f2.Invalid = true
// Should remove
err = m.Update([]protocol.FileInfo{f1, f2, f3})
@@ -145,9 +147,15 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
t.Fatal("db not empty")
}
f1.Flags = 0
f2.Flags = 0
f3.Flags = 0
f1.Deleted = false
f1.Invalid = false
f1.Permissions = 0
f2.Deleted = false
f2.Invalid = false
f2.Permissions = 0
f3.Deleted = false
f3.Invalid = false
f3.Permissions = 0
}
func TestBlockFinderLookup(t *testing.T) {
@@ -187,7 +195,7 @@ func TestBlockFinderLookup(t *testing.T) {
t.Fatal("Incorrect count", counter)
}
f1.Flags |= protocol.FlagDeleted
f1.Deleted = true
err = m1.Update([]protocol.FileInfo{f1})
if err != nil {
@@ -212,7 +220,7 @@ func TestBlockFinderLookup(t *testing.T) {
t.Fatal("Incorrect count")
}
f1.Flags = 0
f1.Deleted = false
}
func TestBlockFinderFix(t *testing.T) {

20
lib/db/leveldb.go
View File

@@ -4,9 +4,6 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o leveldb_xdr.go leveldb.go
package db
import (
@@ -46,25 +43,16 @@ const (
KeyTypeDeviceIdx
)
type fileVersion struct {
version protocol.Vector
device []byte
}
type VersionList struct {
versions []fileVersion
}
func (l VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range l.versions {
for i, v := range l.Versions {
if i > 0 {
b.WriteString(", ")
}
copy(id[:], v.device)
fmt.Fprintf(&b, "{%d, %v}", v.version, id)
copy(id[:], v.Device)
fmt.Fprintf(&b, "{%d, %v}", v.Version, id)
}
b.WriteString("}")
return b.String()
@@ -101,7 +89,7 @@ func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) {
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
err = f.Unmarshal(bs)
if err != nil {
panic(err)
}

114
lib/db/leveldb_convert.go
View File

@@ -1,114 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"bytes"
"github.com/syndtr/goleveldb/leveldb"
)
// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
// avoid having to do rescan. The change is in the key format for folder
// labels, so we basically just iterate over the database rewriting keys as
// necessary.
func convertKeyFormat(from, to *leveldb.DB) error {
l.Infoln("Converting database key format")
blocks, files, globals, unchanged := 0, 0, 0, 0
dbi := newDBInstance(to)
i := from.NewIterator(nil, nil)
for i.Next() {
key := i.Key()
switch key[0] {
case KeyTypeBlock:
folder, file := oldFromBlockKey(key)
folderIdx := dbi.folderIdx.ID([]byte(folder))
hash := key[1+64:]
newKey := blockKeyInto(nil, hash, folderIdx, file)
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
blocks++
case KeyTypeDevice:
newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
files++
case KeyTypeGlobal:
newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
if err := to.Put(newKey, i.Value(), nil); err != nil {
return err
}
globals++
case KeyTypeVirtualMtime:
// Cannot be converted, we drop it instead :(
default:
if err := to.Put(key, i.Value(), nil); err != nil {
return err
}
unchanged++
}
}
l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)
return nil
}
func oldDeviceKeyFolder(key []byte) []byte {
folder := key[1 : 1+64]
izero := bytes.IndexByte(folder, 0)
if izero < 0 {
return folder
}
return folder[:izero]
}
func oldDeviceKeyDevice(key []byte) []byte {
return key[1+64 : 1+64+32]
}
func oldDeviceKeyName(key []byte) []byte {
return key[1+64+32:]
}
func oldGlobalKeyName(key []byte) []byte {
return key[1+64:]
}
func oldGlobalKeyFolder(key []byte) []byte {
folder := key[1 : 1+64]
izero := bytes.IndexByte(folder, 0)
if izero < 0 {
return folder
}
return folder[:izero]
}
func oldFromBlockKey(data []byte) (string, string) {
if len(data) < 1+64+32+1 {
panic("Incorrect key length")
}
if data[0] != KeyTypeBlock {
panic("Incorrect key type")
}
file := string(data[1+64+32:])
slice := data[1 : 1+64]
izero := bytes.IndexByte(slice, 0)
if izero > -1 {
return string(slice[:izero]), file
}
return string(slice), file
}

136
lib/db/leveldb_convert_test.go
View File

@@ -1,136 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"archive/zip"
"io"
"os"
"path/filepath"
"testing"
"github.com/syndtr/goleveldb/leveldb"
)
func TestLabelConversion(t *testing.T) {
os.RemoveAll("testdata/oldformat.db")
defer os.RemoveAll("testdata/oldformat.db")
os.RemoveAll("testdata/newformat.db")
defer os.RemoveAll("testdata/newformat.db")
if err := unzip("testdata/oldformat.db.zip", "testdata"); err != nil {
t.Fatal(err)
}
odb, err := leveldb.OpenFile("testdata/oldformat.db", nil)
if err != nil {
t.Fatal(err)
}
ldb, err := leveldb.OpenFile("testdata/newformat.db", nil)
if err != nil {
t.Fatal(err)
}
if err = convertKeyFormat(odb, ldb); err != nil {
t.Fatal(err)
}
ldb.Close()
odb.Close()
inst, err := Open("testdata/newformat.db")
if err != nil {
t.Fatal(err)
}
fs := NewFileSet("default", inst)
files, deleted, _ := fs.GlobalSize()
if files+deleted != 953 {
// Expected number of global entries determined by
// ../../bin/stindex testdata/oldformat.db/ | grep global | grep -c default
t.Errorf("Conversion error, global list differs (%d != 953)", files+deleted)
}
files, deleted, _ = fs.LocalSize()
if files+deleted != 953 {
t.Errorf("Conversion error, device list differs (%d != 953)", files+deleted)
}
f := NewBlockFinder(inst)
// [block] F:"default" H:1c25dea9003cc16216e2a22900be1ec1cc5aaf270442904e2f9812c314e929d8 N:"f/f2/f25f1b3e6e029231b933531b2138796d" I:3
h := []byte{0x1c, 0x25, 0xde, 0xa9, 0x00, 0x3c, 0xc1, 0x62, 0x16, 0xe2, 0xa2, 0x29, 0x00, 0xbe, 0x1e, 0xc1, 0xcc, 0x5a, 0xaf, 0x27, 0x04, 0x42, 0x90, 0x4e, 0x2f, 0x98, 0x12, 0xc3, 0x14, 0xe9, 0x29, 0xd8}
found := 0
f.Iterate([]string{"default"}, h, func(folder, file string, idx int32) bool {
if folder == "default" && file == filepath.FromSlash("f/f2/f25f1b3e6e029231b933531b2138796d") && idx == 3 {
found++
}
return true
})
if found != 1 {
t.Errorf("Found %d blocks instead of expected 1", found)
}
inst.Close()
}
func unzip(src, dest string) error {
r, err := zip.OpenReader(src)
if err != nil {
return err
}
defer func() {
if err := r.Close(); err != nil {
panic(err)
}
}()
os.MkdirAll(dest, 0755)
// Closure to address file descriptors issue with all the deferred .Close() methods
extractAndWriteFile := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
panic(err)
}
}()
path := filepath.Join(dest, f.Name)
if f.FileInfo().IsDir() {
os.MkdirAll(path, f.Mode())
} else {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
panic(err)
}
}()
_, err = io.Copy(f, rc)
if err != nil {
return err
}
}
return nil
}
for _, f := range r.File {
err := extractAndWriteFile(f)
if err != nil {
return err
}
}
return nil
}

135
lib/db/leveldb_dbinstance.go
View File

@@ -10,12 +10,10 @@ import (
"bytes"
"encoding/binary"
"os"
"path/filepath"
"sort"
"strings"
"sync/atomic"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syndtr/goleveldb/leveldb"
@@ -48,16 +46,6 @@ func Open(file string) (*Instance, error) {
WriteBuffer: 4 << 20,
}
if _, err := os.Stat(file); os.IsNotExist(err) {
// The file we are looking to open does not exist. This may be the
// first launch so we should look for an old version and try to
// convert it.
if err := checkConvertDatabase(file); err != nil {
l.Infoln("Converting old database:", err)
l.Infoln("Will rescan from scratch.")
}
}
db, err := leveldb.OpenFile(file, opts)
if leveldbIsCorrupted(err) {
db, err = leveldb.RecoverFile(file, opts)
@@ -151,12 +139,12 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
case moreFs && moreDb && cmp == 0:
// File exists on both sides - compare versions. We might get an
// update with the same version and different flags if a device has
// marked a file as invalid, so handle that too.
// update with the same version if a device has marked a file as
// invalid, so handle that too.
l.Debugln("generic replace; exists - compare")
var ef FileInfoTruncated
ef.UnmarshalXDR(dbi.Value())
if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
ef.Unmarshal(dbi.Value())
if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Invalid != ef.Invalid {
l.Debugln("generic replace; differs - insert")
if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
maxLocalVer = lv
@@ -232,13 +220,12 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
}
var ef FileInfoTruncated
err = ef.UnmarshalXDR(bs)
err = ef.Unmarshal(bs)
if err != nil {
panic(err)
}
// Flags might change without the version being bumped when we set the
// invalid flag on an existing file.
if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
// The Invalid flag might change without the version being bumped.
if !ef.Version.Equal(f.Version) || ef.Invalid != f.Invalid {
if isLocalDevice {
localSize.removeFile(ef)
localSize.addFile(f)
@@ -308,7 +295,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
// struct, which in turn references the buffer it was unmarshalled
// from. dbi.Value() just returns an internal slice that it reuses, so
// we need to copy it.
err := f.UnmarshalXDR(append([]byte{}, dbi.Value()...))
err := f.Unmarshal(append([]byte{}, dbi.Value()...))
if err != nil {
panic(err)
}
@@ -347,16 +334,16 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
}
var vl VersionList
err = vl.UnmarshalXDR(bs)
err = vl.Unmarshal(bs)
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(k)
panic("no versions?")
}
k = db.deviceKey(folder, vl.versions[0].device, file)
k = db.deviceKey(folder, vl.Versions[0].Device, file)
bs, err = t.Get(k, nil)
if err != nil {
panic(err)
@@ -384,11 +371,11 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
var fk []byte
for dbi.Next() {
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
@@ -398,13 +385,13 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
return
}
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[0].Device, name)
bs, err := t.Get(fk, nil)
if err != nil {
l.Debugf("folder: %q (%x)", folder, folder)
l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
l.Debugf("vl: %v", vl)
l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
l.Debugf("vl.Versions[0].Device: %x", vl.Versions[0].Device)
l.Debugf("name: %q (%x)", name, name)
l.Debugf("fk: %q", fk)
l.Debugf("fk: %x %x %x",
@@ -436,17 +423,17 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
}
var vl VersionList
err = vl.UnmarshalXDR(bs)
err = vl.Unmarshal(bs)
if err != nil {
panic(err)
}
var devices []protocol.DeviceID
for _, v := range vl.versions {
if !v.version.Equal(vl.versions[0].version) {
for _, v := range vl.Versions {
if !v.Version.Equal(vl.Versions[0].Version) {
break
}
n := protocol.DeviceIDFromBytes(v.device)
n := protocol.DeviceIDFromBytes(v.Device)
devices = append(devices, n)
}
@@ -464,11 +451,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
nextFile:
for dbi.Next() {
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
if len(vl.Versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
@@ -476,29 +463,29 @@ nextFile:
have := false // If we have the file, any version
need := false // If we have a lower version of the file
var haveVersion protocol.Vector
for _, v := range vl.versions {
if bytes.Equal(v.device, device) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
have = true
haveVersion = v.version
haveVersion = v.Version
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
need = !v.version.GreaterEqual(vl.versions[0].version)
need = !v.Version.GreaterEqual(vl.Versions[0].Version)
break
}
}
if need || !have {
name := db.globalKeyName(dbi.Key())
needVersion := vl.versions[0].version
needVersion := vl.Versions[0].Version
nextVersion:
for i := range vl.versions {
if !vl.versions[i].version.Equal(needVersion) {
for i := range vl.Versions {
if !vl.Versions[i].Version.Equal(needVersion) {
// We haven't found a valid copy of the file with the needed version.
continue nextFile
}
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[i].Device, name)
bs, err := t.Get(fk, nil)
if err != nil {
var id protocol.DeviceID
@@ -528,7 +515,7 @@ nextFile:
continue nextFile
}
l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.Versions[0].Version)
if cont := fn(gf); !cont {
return
@@ -601,7 +588,7 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
for dbi.Next() {
gk := dbi.Key()
var vl VersionList
err := vl.UnmarshalXDR(dbi.Value())
err := vl.Unmarshal(dbi.Value())
if err != nil {
panic(err)
}
@@ -613,8 +600,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
name := db.globalKeyName(gk)
var newVL VersionList
for i, version := range vl.versions {
fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
for i, version := range vl.Versions {
fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.Device, name)
_, err := t.Get(fk, nil)
if err == leveldb.ErrNotFound {
@@ -623,10 +610,10 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
if err != nil {
panic(err)
}
newVL.versions = append(newVL.versions, version)
newVL.Versions = append(newVL.Versions, version)
if i == 0 {
fi, ok := t.getFile(folder, version.device, name)
fi, ok := t.getFile(folder, version.Device, name)
if !ok {
panic("nonexistent global master file")
}
@@ -634,8 +621,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
}
}
if len(newVL.versions) != len(vl.versions) {
t.Put(dbi.Key(), newVL.MustMarshalXDR())
if len(newVL.Versions) != len(vl.Versions) {
t.Put(dbi.Key(), mustMarshal(&newVL))
t.checkFlush()
}
}
@@ -715,12 +702,12 @@ func (db *Instance) globalKeyFolder(key []byte) []byte {
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf FileInfoTruncated
err := tf.UnmarshalXDR(bs)
err := tf.Unmarshal(bs)
return tf, err
}
var tf protocol.FileInfo
err := tf.UnmarshalXDR(bs)
err := tf.Unmarshal(bs)
return tf, err
}
@@ -740,50 +727,6 @@ func leveldbIsCorrupted(err error) bool {
return false
}
// checkConvertDatabase tries to convert an existing old (v0.11) database to
// new (v0.13) format.
func checkConvertDatabase(dbFile string) error {
oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db")
if _, err := os.Stat(oldLoc); os.IsNotExist(err) {
// The old database file does not exist; that's ok, continue as if
// everything succeeded.
return nil
} else if err != nil {
// Any other error is weird.
return err
}
// There exists a database in the old format. We run a one time
// conversion from old to new.
fromDb, err := leveldb.OpenFile(oldLoc, nil)
if err != nil {
return err
}
toDb, err := leveldb.OpenFile(dbFile, nil)
if err != nil {
return err
}
err = convertKeyFormat(fromDb, toDb)
if err != nil {
return err
}
err = toDb.Close()
if err != nil {
return err
}
// We've done this one, we don't want to do it again (if the user runs
// -reset or so). We don't care too much about errors any more at this stage.
fromDb.Close()
osutil.Rename(oldLoc, oldLoc+".converted")
return nil
}
// A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
// fast lookups in both directions and persists to the database. Don't use for
// storing more items than fit comfortably in RAM.

70
lib/db/leveldb_transactions.go
View File

@@ -83,7 +83,7 @@ func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.Fi
name := []byte(file.Name)
nk := t.db.deviceKey(folder, device, name)
t.Put(nk, file.MustMarshalXDR())
t.Put(nk, mustMarshal(&file))
return file.LocalVersion
}
@@ -105,14 +105,14 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
var hasOldFile bool
// Remove the device from the current version list
if len(svl) != 0 {
err = fl.UnmarshalXDR(svl)
err = fl.Unmarshal(svl)
if err != nil {
panic(err)
}
for i := range fl.versions {
if bytes.Equal(fl.versions[i].device, device) {
if fl.versions[i].version.Equal(file.Version) {
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if fl.Versions[i].Version.Equal(file.Version) {
// No need to do anything
return false
}
@@ -120,29 +120,29 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
if i == 0 {
// Keep the current newest file around so we can subtract it from
// the globalSize if we replace it.
oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name)
oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
}
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
break
}
}
}
nv := fileVersion{
device: device,
version: file.Version,
nv := FileVersion{
Device: device,
Version: file.Version,
}
insertedAt := -1
// Find a position in the list to insert this file. The file at the front
// of the list is the newer, the "global".
for i := range fl.versions {
switch fl.versions[i].version.Compare(file.Version) {
for i := range fl.Versions {
switch fl.Versions[i].Version.Compare(file.Version) {
case protocol.Equal, protocol.Lesser:
// The version at this point in the list is equal to or lesser
// ("older") than us. We insert ourselves in front of it.
fl.versions = insertVersion(fl.versions, i, nv)
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
goto done
@@ -153,12 +153,12 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
// "Greater" in the condition above is just based on the device
// IDs in the version vector, which is not the only thing we use
// to determine the winner.)
of, ok := t.getFile(folder, fl.versions[i].device, name)
of, ok := t.getFile(folder, fl.Versions[i].Device, name)
if !ok {
panic("file referenced in version list does not exist")
}
if file.WinsConflict(of) {
fl.versions = insertVersion(fl.versions, i, nv)
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
goto done
}
@@ -166,8 +166,8 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
}
// We didn't find a position for an insert above, so append to the end.
fl.versions = append(fl.versions, nv)
insertedAt = len(fl.versions) - 1
fl.Versions = append(fl.Versions, nv)
insertedAt = len(fl.Versions) - 1
done:
if insertedAt == 0 {
@@ -178,9 +178,9 @@ done:
if hasOldFile {
// We have the old file that was removed at the head of the list.
globalSize.removeFile(oldFile)
} else if len(fl.versions) > 1 {
} else if len(fl.Versions) > 1 {
// The previous newest version is now at index 1, grab it from there.
oldFile, ok := t.getFile(folder, fl.versions[1].device, name)
oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name)
if !ok {
panic("file referenced in version list does not exist")
}
@@ -190,7 +190,7 @@ done:
}
l.Debugf("new global after update: %v", fl)
t.Put(gk, fl.MustMarshalXDR())
t.Put(gk, mustMarshal(&fl))
return true
}
@@ -210,14 +210,14 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
}
var fl VersionList
err = fl.UnmarshalXDR(svl)
err = fl.Unmarshal(svl)
if err != nil {
panic(err)
}
removed := false
for i := range fl.versions {
if bytes.Equal(fl.versions[i].device, device) {
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if i == 0 && globalSize != nil {
f, ok := t.getFile(folder, device, file)
if !ok {
@@ -226,18 +226,18 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
globalSize.removeFile(f)
removed = true
}
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
break
}
}
if len(fl.versions) == 0 {
if len(fl.Versions) == 0 {
t.Delete(gk)
} else {
l.Debugf("new global after remove: %v", fl)
t.Put(gk, fl.MustMarshalXDR())
t.Put(gk, mustMarshal(&fl))
if removed {
f, ok := t.getFile(folder, fl.versions[0].device, file)
f, ok := t.getFile(folder, fl.Versions[0].Device, file)
if !ok {
panic("new global is nonexistent file")
}
@@ -246,9 +246,21 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
}
}
func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
t := append(vl, fileVersion{})
func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
t := append(vl, FileVersion{})
copy(t[i+1:], t[i:])
t[i] = v
return t
}
type marshaller interface {
Marshal() ([]byte, error)
}
func mustMarshal(f marshaller) []byte {
bs, err := f.Marshal()
if err != nil {
panic(err)
}
return bs
}

142
lib/db/leveldb_xdr.go
View File

@@ -1,142 +0,0 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package db
import (
"github.com/calmh/xdr"
)
/*
fileVersion Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Vector Structure \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ device (length + padded data) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct fileVersion {
Vector version;
opaque device<>;
}
*/
func (o fileVersion) XDRSize() int {
return o.version.XDRSize() +
4 + len(o.device) + xdr.Padding(len(o.device))
}
func (o fileVersion) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o fileVersion) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o fileVersion) MarshalXDRInto(m *xdr.Marshaller) error {
if err := o.version.MarshalXDRInto(m); err != nil {
return err
}
m.MarshalBytes(o.device)
return m.Error
}
func (o *fileVersion) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *fileVersion) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
(&o.version).UnmarshalXDRFrom(u)
o.device = u.UnmarshalBytes()
return u.Error
}
/*
VersionList Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of versions |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more fileVersion Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct VersionList {
fileVersion versions<>;
}
*/
func (o VersionList) XDRSize() int {
return 4 + xdr.SizeOfSlice(o.versions)
}
func (o VersionList) MarshalXDR() ([]byte, error) {
buf := make([]byte, o.XDRSize())
m := &xdr.Marshaller{Data: buf}
return buf, o.MarshalXDRInto(m)
}
func (o VersionList) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o VersionList) MarshalXDRInto(m *xdr.Marshaller) error {
m.MarshalUint32(uint32(len(o.versions)))
for i := range o.versions {
if err := o.versions[i].MarshalXDRInto(m); err != nil {
return err
}
}
return m.Error
}
func (o *VersionList) UnmarshalXDR(bs []byte) error {
u := &xdr.Unmarshaller{Data: bs}
return o.UnmarshalXDRFrom(u)
}
func (o *VersionList) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
_versionsSize := int(u.UnmarshalUint32())
if _versionsSize < 0 {
return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
} else if _versionsSize == 0 {
o.versions = nil
} else {
if _versionsSize <= len(o.versions) {
o.versions = o.versions[:_versionsSize]
} else {
o.versions = make([]fileVersion, _versionsSize)
}
for i := range o.versions {
(&o.versions[i]).UnmarshalXDRFrom(u)
}
}
return u.Error
}

11
lib/db/set.go
View File

@@ -31,9 +31,10 @@ type FileSet struct {
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
// FileInfoTruncated.
type FileIntf interface {
Size() int64
FileSize() int64
FileName() string
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
@@ -42,7 +43,7 @@ type FileIntf interface {
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
@@ -64,7 +65,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
} else {
s.files++
}
s.bytes += f.Size()
s.bytes += f.FileSize()
s.mut.Unlock()
}
@@ -79,7 +80,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
} else {
s.files--
}
s.bytes -= f.Size()
s.bytes -= f.FileSize()
if s.deleted < 0 || s.files < 0 {
panic("bug: removed more than added")
}

178
lib/db/set_test.go
View File

@@ -88,7 +88,7 @@ func (l fileList) String() string {
var b bytes.Buffer
b.WriteString("[]protocol.FileList{\n")
for _, f := range l {
fmt.Fprintf(&b, " %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
fmt.Fprintf(&b, " %q: #%d, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
}
b.WriteString("}")
return b.String()
@@ -100,35 +100,35 @@ func TestGlobalSet(t *testing.T) {
m := db.NewFileSet("test", ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
}
local1 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
}
localTot := fileList{
local0[0],
local0[1],
local0[2],
local0[3],
protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
}
remote0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
}
remote1 := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
}
remoteTot := fileList{
remote0[0],
@@ -178,7 +178,7 @@ func TestGlobalSet(t *testing.T) {
} else {
globalFiles++
}
globalBytes += f.Size()
globalBytes += f.FileSize()
}
gsFiles, gsDeleted, gsBytes := m.GlobalSize()
if gsFiles != globalFiles {
@@ -208,7 +208,7 @@ func TestGlobalSet(t *testing.T) {
} else {
haveFiles++
}
haveBytes += f.Size()
haveBytes += f.FileSize()
}
lsFiles, lsDeleted, lsBytes := m.LocalSize()
if lsFiles != haveFiles {
@@ -303,23 +303,23 @@ func TestNeedWithInvalid(t *testing.T) {
s := db.NewFileSet("test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
}
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
remote1Have := fileList{
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
expectedNeed := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
s.Replace(protocol.LocalDeviceID, localHave)
@@ -340,10 +340,10 @@ func TestUpdateToInvalid(t *testing.T) {
s := db.NewFileSet("test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
s.Replace(protocol.LocalDeviceID, localHave)
@@ -355,7 +355,7 @@ func TestUpdateToInvalid(t *testing.T) {
t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
}
localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Invalid: true}
s.Update(protocol.LocalDeviceID, localHave[1:2])
have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -372,16 +372,16 @@ func TestInvalidAvailability(t *testing.T) {
s := db.NewFileSet("test", ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
remote1Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
}
s.Replace(remoteDevice0, remote0Have)
@@ -410,17 +410,17 @@ func TestGlobalReset(t *testing.T) {
m := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local)
@@ -448,23 +448,23 @@ func TestNeed(t *testing.T) {
m := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
shouldNeed := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local)
@@ -486,18 +486,18 @@ func TestLocalVersion(t *testing.T) {
m := db.NewFileSet("test", ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
local2 := []protocol.FileInfo{
local1[0],
// [1] deleted
local1[2],
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
m.Replace(protocol.LocalDeviceID, local1)
@@ -515,17 +515,17 @@ func TestListDropFolder(t *testing.T) {
s0 := db.NewFileSet("test0", ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
s0.Replace(protocol.LocalDeviceID, local1)
s1 := db.NewFileSet("test1", ldb)
local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
}
s1.Replace(remoteDevice0, local2)
@@ -566,24 +566,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
s := db.NewFileSet("test1", ldb)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
}
s.Replace(remoteDevice0, rem0)
rem1 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
}
s.Replace(remoteDevice1, rem1)
total := fileList{
// There's a valid copy of each file, so it should be merged
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
}
need := fileList(needList(s, protocol.LocalDeviceID))
@@ -609,7 +609,7 @@ func TestLongPath(t *testing.T) {
name := b.String() // 5000 characters
local := []protocol.FileInfo{
{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
s.Replace(protocol.LocalDeviceID, local)
@@ -633,7 +633,7 @@ func TestCommitted(t *testing.T) {
s := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
// Adding a file should increase the counter
@@ -659,12 +659,12 @@ func TestCommitted(t *testing.T) {
func BenchmarkUpdateOneFile(b *testing.B) {
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
// A longer name is more realistic and causes more allocations
protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
}
ldb, err := db.Open("testdata/benchmarkupdate.db")

57
lib/db/structs.go Normal file
View File

@@ -0,0 +1,57 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:generate go run ../../script/protofmt.go structs.proto
//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. structs.proto
package db
import (
"fmt"
"github.com/syncthing/syncthing/lib/protocol"
)
func (f FileInfoTruncated) String() string {
return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
}
func (f FileInfoTruncated) IsDeleted() bool {
return f.Deleted
}
func (f FileInfoTruncated) IsInvalid() bool {
return f.Invalid
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Type == protocol.FileInfoTypeDirectory
}
func (f FileInfoTruncated) IsSymlink() bool {
switch f.Type {
case protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile, protocol.FileInfoTypeSymlinkUnknown:
return true
default:
return false
}
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return !f.NoPermissions
}
func (f FileInfoTruncated) FileSize() int64 {
if f.IsDirectory() || f.IsDeleted() {
return 128
}
return f.Size
}
func (f FileInfoTruncated) FileName() string {
return f.Name
}

914
lib/db/structs.pb.go Normal file
View File

@@ -0,0 +1,914 @@
// Code generated by protoc-gen-gogo.
// source: structs.proto
// DO NOT EDIT!
/*
Package db is a generated protocol buffer package.
It is generated from these files:
structs.proto
It has these top-level messages:
FileVersion
VersionList
FileInfoTruncated
*/
package db
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import protocol "github.com/syncthing/syncthing/lib/protocol"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
const _ = proto.GoGoProtoPackageIsVersion1
type FileVersion struct {
Version protocol.Vector `protobuf:"bytes,1,opt,name=version" json:"version"`
Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
}
func (m *FileVersion) Reset() { *m = FileVersion{} }
func (m *FileVersion) String() string { return proto.CompactTextString(m) }
func (*FileVersion) ProtoMessage() {}
func (*FileVersion) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{0} }
type VersionList struct {
Versions []FileVersion `protobuf:"bytes,1,rep,name=versions" json:"versions"`
}
func (m *VersionList) Reset() { *m = VersionList{} }
func (*VersionList) ProtoMessage() {}
func (*VersionList) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{1} }
// Must be the same as FileInfo but without the blocks field
type FileInfoTruncated struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Type protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
Size int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
Permissions uint32 `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
Modified int64 `protobuf:"varint,5,opt,name=modified,proto3" json:"modified,omitempty"`
Deleted bool `protobuf:"varint,6,opt,name=deleted,proto3" json:"deleted,omitempty"`
Invalid bool `protobuf:"varint,7,opt,name=invalid,proto3" json:"invalid,omitempty"`
NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
Version protocol.Vector `protobuf:"bytes,9,opt,name=version" json:"version"`
LocalVersion int64 `protobuf:"varint,10,opt,name=local_version,json=localVersion,proto3" json:"local_version,omitempty"`
}
func (m *FileInfoTruncated) Reset() { *m = FileInfoTruncated{} }
func (*FileInfoTruncated) ProtoMessage() {}
func (*FileInfoTruncated) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{2} }
func init() {
proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
proto.RegisterType((*VersionList)(nil), "db.VersionList")
proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
}
func (m *FileVersion) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *FileVersion) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
n1, err := m.Version.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n1
if len(m.Device) > 0 {
data[i] = 0x12
i++
i = encodeVarintStructs(data, i, uint64(len(m.Device)))
i += copy(data[i:], m.Device)
}
return i, nil
}
func (m *VersionList) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *VersionList) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Versions) > 0 {
for _, msg := range m.Versions {
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(msg.ProtoSize()))
n, err := msg.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n
}
}
return i, nil
}
func (m *FileInfoTruncated) Marshal() (data []byte, err error) {
size := m.ProtoSize()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *FileInfoTruncated) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Name) > 0 {
data[i] = 0xa
i++
i = encodeVarintStructs(data, i, uint64(len(m.Name)))
i += copy(data[i:], m.Name)
}
if m.Type != 0 {
data[i] = 0x10
i++
i = encodeVarintStructs(data, i, uint64(m.Type))
}
if m.Size != 0 {
data[i] = 0x18
i++
i = encodeVarintStructs(data, i, uint64(m.Size))
}
if m.Permissions != 0 {
data[i] = 0x20
i++
i = encodeVarintStructs(data, i, uint64(m.Permissions))
}
if m.Modified != 0 {
data[i] = 0x28
i++
i = encodeVarintStructs(data, i, uint64(m.Modified))
}
if m.Deleted {
data[i] = 0x30
i++
if m.Deleted {
data[i] = 1
} else {
data[i] = 0
}
i++
}
if m.Invalid {
data[i] = 0x38
i++
if m.Invalid {
data[i] = 1
} else {
data[i] = 0
}
i++
}
if m.NoPermissions {
data[i] = 0x40
i++
if m.NoPermissions {
data[i] = 1
} else {
data[i] = 0
}
i++
}
data[i] = 0x4a
i++
i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
n2, err := m.Version.MarshalTo(data[i:])
if err != nil {
return 0, err
}
i += n2
if m.LocalVersion != 0 {
data[i] = 0x50
i++
i = encodeVarintStructs(data, i, uint64(m.LocalVersion))
}
return i, nil
}
func encodeFixed64Structs(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Structs(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintStructs(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *FileVersion) ProtoSize() (n int) {
var l int
_ = l
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
l = len(m.Device)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
return n
}
func (m *VersionList) ProtoSize() (n int) {
var l int
_ = l
if len(m.Versions) > 0 {
for _, e := range m.Versions {
l = e.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
}
}
return n
}
func (m *FileInfoTruncated) ProtoSize() (n int) {
var l int
_ = l
l = len(m.Name)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
if m.Type != 0 {
n += 1 + sovStructs(uint64(m.Type))
}
if m.Size != 0 {
n += 1 + sovStructs(uint64(m.Size))
}
if m.Permissions != 0 {
n += 1 + sovStructs(uint64(m.Permissions))
}
if m.Modified != 0 {
n += 1 + sovStructs(uint64(m.Modified))
}
if m.Deleted {
n += 2
}
if m.Invalid {
n += 2
}
if m.NoPermissions {
n += 2
}
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
if m.LocalVersion != 0 {
n += 1 + sovStructs(uint64(m.LocalVersion))
}
return n
}
func sovStructs(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozStructs(x uint64) (n int) {
return sovStructs(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *FileVersion) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: FileVersion: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: FileVersion: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Device = append(m.Device[:0], data[iNdEx:postIndex]...)
if m.Device == nil {
m.Device = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *VersionList) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: VersionList: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: VersionList: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Versions = append(m.Versions, FileVersion{})
if err := m.Versions[len(m.Versions)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *FileInfoTruncated) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: FileInfoTruncated: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: FileInfoTruncated: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Name = string(data[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
}
m.Type = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Type |= (protocol.FileInfoType(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType)
}
m.Size = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Size |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Permissions", wireType)
}
m.Permissions = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Permissions |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Modified", wireType)
}
m.Modified = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Modified |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Deleted = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.Invalid = bool(v != 0)
case 8:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field NoPermissions", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.NoPermissions = bool(v != 0)
case 9:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 10:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalVersion", wireType)
}
m.LocalVersion = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.LocalVersion |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipStructs(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipStructs(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthStructs
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowStructs
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipStructs(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthStructs = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowStructs = fmt.Errorf("proto: integer overflow")
)
var fileDescriptorStructs = []byte{
// 401 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x4f, 0xcb, 0xd3, 0x30,
0x1c, 0x6e, 0xb7, 0xba, 0xf5, 0x4d, 0xdf, 0x4e, 0x0d, 0x32, 0xca, 0x0e, 0xdd, 0x98, 0x08, 0x22,
0xd8, 0xe9, 0xc4, 0x8b, 0xc7, 0x1d, 0x06, 0x82, 0x07, 0x29, 0x32, 0x8f, 0xa3, 0x4d, 0xb2, 0x2e,
0xd0, 0x26, 0xa5, 0x49, 0x07, 0xf3, 0x93, 0x78, 0xdc, 0xc7, 0xd9, 0xd1, 0x2f, 0xa0, 0xe8, 0xfc,
0x22, 0x66, 0x49, 0x3b, 0x7b, 0x7c, 0x0f, 0x81, 0xdf, 0x93, 0xe7, 0xcf, 0xef, 0x21, 0x01, 0xbe,
0x90, 0x55, 0x8d, 0xa4, 0x88, 0xca, 0x8a, 0x4b, 0x0e, 0x7b, 0x38, 0x9d, 0xbc, 0xce, 0xa8, 0xdc,
0xd7, 0x69, 0x84, 0x78, 0xb1, 0xc8, 0x78, 0xc6, 0x17, 0x9a, 0x4a, 0xeb, 0x9d, 0x46, 0x1a, 0xe8,
0xc9, 0x58, 0x26, 0xef, 0x3b, 0x72, 0x71, 0x64, 0x48, 0xee, 0x29, 0xcb, 0x3a, 0x53, 0x4e, 0x53,
0x93, 0x80, 0x78, 0xbe, 0x48, 0x49, 0x69, 0x6c, 0xf3, 0xaf, 0xc0, 0x5b, 0xd3, 0x9c, 0x6c, 0x48,
0x25, 0x28, 0x67, 0xf0, 0x0d, 0x18, 0x1e, 0xcc, 0x18, 0xd8, 0x33, 0xfb, 0xa5, 0xb7, 0x7c, 0x12,
0xb5, 0xa6, 0x68, 0x43, 0x90, 0xe4, 0xd5, 0xca, 0x39, 0xff, 0x9a, 0x5a, 0x71, 0x2b, 0x83, 0x63,
0x30, 0xc0, 0xe4, 0x40, 0x11, 0x09, 0x7a, 0xca, 0x70, 0x1f, 0x37, 0x68, 0xbe, 0x06, 0x5e, 0x13,
0xfa, 0x89, 0x0a, 0x09, 0xdf, 0x02, 0xb7, 0x71, 0x08, 0x95, 0xdc, 0x57, 0xc9, 0x8f, 0x23, 0x9c,
0x46, 0x9d, 0xdd, 0x4d, 0xf0, 0x4d, 0xf6, 0xc1, 0xf9, 0x7e, 0x9a, 0x5a, 0xf3, 0x9f, 0x3d, 0xf0,
0xf4, 0xaa, 0xfa, 0xc8, 0x76, 0xfc, 0x4b, 0x55, 0x33, 0x94, 0x48, 0x82, 0x21, 0x04, 0x0e, 0x4b,
0x0a, 0xa2, 0x4b, 0xde, 0xc5, 0x7a, 0x86, 0xaf, 0x80, 0x23, 0x8f, 0xa5, 0xe9, 0x31, 0x5a, 0x8e,
0xff, 0x17, 0xbf, 0xd9, 0x15, 0x1b, 0x6b, 0xcd, 0xd5, 0x2f, 0xe8, 0x37, 0x12, 0xf4, 0x95, 0xb6,
0x1f, 0xeb, 0x19, 0xce, 0x80, 0x57, 0x92, 0xaa, 0xa0, 0xc2, 0xb4, 0x74, 0x14, 0xe5, 0xc7, 0xdd,
0x2b, 0x38, 0x01, 0x6e, 0xc1, 0x31, 0xdd, 0x51, 0x82, 0x83, 0x47, 0xda, 0x79, 0xc3, 0x30, 0x00,
0x43, 0x4c, 0x72, 0xa2, 0xca, 0x05, 0x03, 0x45, 0xb9, 0x71, 0x0b, 0xaf, 0x0c, 0x65, 0x87, 0x24,
0xa7, 0x38, 0x18, 0x1a, 0xa6, 0x81, 0xf0, 0x05, 0x18, 0x31, 0xbe, 0xed, 0x2e, 0x75, 0xb5, 0xc0,
0x67, 0xfc, 0x73, 0x67, 0x6d, 0xe7, 0x53, 0xee, 0x1e, 0xf6, 0x29, 0xcf, 0x81, 0x9f, 0x73, 0x94,
0xe4, 0xdb, 0xd6, 0x07, 0x74, 0xdb, 0x7b, 0x7d, 0xd9, 0xbc, 0xb7, 0x79, 0xdf, 0xd5, 0xb3, 0xf3,
0x9f, 0xd0, 0x3a, 0x5f, 0x42, 0xfb, 0x87, 0x3a, 0xbf, 0x2f, 0xa1, 0x75, 0xfa, 0x1b, 0xda, 0xe9,
0x40, 0x2f, 0x78, 0xf7, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xb1, 0x7f, 0x07, 0x98, 0x02, 0x00,
0x00,
}

35
lib/db/structs.proto Normal file
View File

@@ -0,0 +1,35 @@
syntax = "proto3";
package db;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "github.com/syncthing/syncthing/lib/protocol/bep.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;
message FileVersion {
protocol.Vector version = 1 [(gogoproto.nullable) = false];
bytes device = 2;
}
message VersionList {
option (gogoproto.goproto_stringer) = false;
repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
}
// Must be the same as FileInfo but without the blocks field
message FileInfoTruncated {
option (gogoproto.goproto_stringer) = false;
string name = 1;
protocol.FileInfoType type = 2;
int64 size = 3;
uint32 permissions = 4;
int64 modified = 5;
bool deleted = 6;
bool invalid = 7;
bool no_permissions = 8;
protocol.Vector version = 9 [(gogoproto.nullable) = false];
int64 local_version = 10;
}

BIN
lib/db/testdata/oldformat.db.zip vendored
View File

Binary file not shown.

52
lib/db/truncated.go
View File

@@ -1,52 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package db
import (
"github.com/calmh/xdr"
"github.com/syncthing/syncthing/lib/protocol"
)
type FileInfoTruncated struct {
protocol.FileInfo
}
func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
return o.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: bs})
}
func (o *FileInfoTruncated) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
o.Name = u.UnmarshalStringMax(8192)
o.Flags = u.UnmarshalUint32()
o.Modified = int64(u.UnmarshalUint64())
(&o.Version).UnmarshalXDRFrom(u)
o.LocalVersion = int64(u.UnmarshalUint64())
_BlocksSize := int(u.UnmarshalUint32())
if _BlocksSize < 0 {
return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
} else if _BlocksSize == 0 {
o.Blocks = nil
} else {
if _BlocksSize > 10000000 {
return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
}
for i := 0; i < _BlocksSize; i++ {
size := int64(u.UnmarshalUint32())
o.CachedSize += size
u.UnmarshalBytes()
}
}
return u.Error
}
func BlocksToSize(num int) int64 {
if num < 2 {
return protocol.BlockSize / 2
}
return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
}