perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move top level packages to internal.

Browse Source
This commit is contained in:
Jakob Borg
2014-09-22 21:42:11 +02:00
parent fbdbd722b1
commit 14817e31f6
174 changed files with 252 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/files/debug.go Normal file
View File

@@ -0,0 +1,17 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package files
import (
"os"
"strings"
"github.com/syncthing/syncthing/internal/logger"
)
var (
debug = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
l = logger.DefaultLogger
)

15
internal/files/filenames_darwin.go Normal file
View File

@@ -0,0 +1,15 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package files
import "code.google.com/p/go.text/unicode/norm"
func normalizedFilename(s string) string {
return norm.NFC.String(s)
}
func nativeFilename(s string) string {
return norm.NFD.String(s)
}

17
internal/files/filenames_unix.go Normal file
View File

@@ -0,0 +1,17 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// +build !windows,!darwin
package files
import "code.google.com/p/go.text/unicode/norm"
func normalizedFilename(s string) string {
return norm.NFC.String(s)
}
func nativeFilename(s string) string {
return s
}

19
internal/files/filenames_windows.go Normal file
View File

@@ -0,0 +1,19 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package files
import (
"path/filepath"
"code.google.com/p/go.text/unicode/norm"
)
func normalizedFilename(s string) string {
return norm.NFC.String(filepath.ToSlash(s))
}
func nativeFilename(s string) string {
return filepath.FromSlash(s)
}

763
internal/files/leveldb.go Normal file
View File

@@ -0,0 +1,763 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package files
import (
"bytes"
"runtime"
"sort"
"sync"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/util"
)
var (
clockTick uint64
clockMut sync.Mutex
)
func clock(v uint64) uint64 {
clockMut.Lock()
defer clockMut.Unlock()
if v > clockTick {
clockTick = v + 1
} else {
clockTick++
}
return clockTick
}
const (
keyTypeNode = iota
keyTypeGlobal
)
type fileVersion struct {
version uint64
node []byte
}
type versionList struct {
versions []fileVersion
}
type fileList []protocol.FileInfo
func (l fileList) Len() int {
return len(l)
}
func (l fileList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func (l fileList) Less(a, b int) bool {
return l[a].Name < l[b].Name
}
type dbReader interface {
Get([]byte, *opt.ReadOptions) ([]byte, error)
}
type dbWriter interface {
Put([]byte, []byte)
Delete([]byte)
}
/*
keyTypeNode (1 byte)
repository (64 bytes)
node (32 bytes)
name (variable size)
|
scanner.File
keyTypeGlobal (1 byte)
repository (64 bytes)
name (variable size)
|
[]fileVersion (sorted)
*/
func nodeKey(repo, node, file []byte) []byte {
k := make([]byte, 1+64+32+len(file))
k[0] = keyTypeNode
copy(k[1:], []byte(repo))
copy(k[1+64:], node[:])
copy(k[1+64+32:], []byte(file))
return k
}
func globalKey(repo, file []byte) []byte {
k := make([]byte, 1+64+len(file))
k[0] = keyTypeGlobal
copy(k[1:], []byte(repo))
copy(k[1+64:], []byte(file))
return k
}
func nodeKeyName(key []byte) []byte {
return key[1+64+32:]
}
func nodeKeyRepo(key []byte) []byte {
repo := key[1 : 1+64]
izero := bytes.IndexByte(repo, 0)
return repo[:izero]
}
func nodeKeyNode(key []byte) []byte {
return key[1+64 : 1+64+32]
}
func globalKeyName(key []byte) []byte {
return key[1+64:]
}
func globalKeyRepo(key []byte) []byte {
repo := key[1 : 1+64]
izero := bytes.IndexByte(repo, 0)
return repo[:izero]
}
type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64
type fileIterator func(f protocol.FileIntf) bool
func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
runtime.GC()
sort.Sort(fileList(fs)) // sort list on name, same as on disk
start := nodeKey(repo, node, nil) // before all repo/node files
limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
batch := new(leveldb.Batch)
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
moreDb := dbi.Next()
fsi := 0
var maxLocalVer uint64
for {
var newName, oldName []byte
moreFs := fsi < len(fs)
if !moreDb && !moreFs {
break
}
if !moreFs && deleteFn == nil {
// We don't have any more updated files to process and deletion
// has not been requested, so we can exit early
break
}
if moreFs {
newName = []byte(fs[fsi].Name)
}
if moreDb {
oldName = nodeKeyName(dbi.Key())
}
cmp := bytes.Compare(newName, oldName)
if debug {
l.Debugf("generic replace; repo=%q node=%v moreFs=%v moreDb=%v cmp=%d newName=%q oldName=%q", repo, protocol.NodeIDFromBytes(node), moreFs, moreDb, cmp, newName, oldName)
}
switch {
case moreFs && (!moreDb || cmp == -1):
// Disk is missing this file. Insert it.
if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
maxLocalVer = lv
}
if fs[fsi].IsInvalid() {
ldbRemoveFromGlobal(snap, batch, repo, node, newName)
} else {
ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
}
fsi++
case moreFs && moreDb && cmp == 0:
// File exists on both sides - compare versions. We might get an
// update with the same version and different flags if a node has
// marked a file as invalid, so handle that too.
var ef protocol.FileInfoTruncated
ef.UnmarshalXDR(dbi.Value())
if fs[fsi].Version > ef.Version || fs[fsi].Version != ef.Version {
if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
maxLocalVer = lv
}
if fs[fsi].IsInvalid() {
ldbRemoveFromGlobal(snap, batch, repo, node, newName)
} else {
ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
}
}
// Iterate both sides.
fsi++
moreDb = dbi.Next()
case moreDb && (!moreFs || cmp == 1):
if deleteFn != nil {
if lv := deleteFn(snap, batch, repo, node, oldName, dbi); lv > maxLocalVer {
maxLocalVer = lv
}
}
moreDb = dbi.Next()
}
}
err = db.Write(batch, nil)
if err != nil {
panic(err)
}
return maxLocalVer
}
func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
// TODO: Return the remaining maxLocalVer?
return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
// Disk has files that we are missing. Remove it.
if debug {
l.Debugf("delete; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
}
ldbRemoveFromGlobal(db, batch, repo, node, name)
batch.Delete(dbi.Key())
return 0
})
}
func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) uint64 {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
}
if !tf.IsDeleted() {
if debug {
l.Debugf("mark deleted; repo=%q node=%v name=%q", repo, protocol.NodeIDFromBytes(node), name)
}
ts := clock(tf.LocalVersion)
f := protocol.FileInfo{
Name: tf.Name,
Version: lamport.Default.Tick(tf.Version),
LocalVersion: ts,
Flags: tf.Flags | protocol.FlagDeleted,
Modified: tf.Modified,
}
batch.Put(dbi.Key(), f.MarshalXDR())
ldbUpdateGlobal(db, batch, repo, node, nodeKeyName(dbi.Key()), f.Version)
return ts
}
return 0
})
}
func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64 {
runtime.GC()
batch := new(leveldb.Batch)
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
var maxLocalVer uint64
for _, f := range fs {
name := []byte(f.Name)
fk := nodeKey(repo, node, name)
bs, err := snap.Get(fk, nil)
if err == leveldb.ErrNotFound {
if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
maxLocalVer = lv
}
if f.IsInvalid() {
ldbRemoveFromGlobal(snap, batch, repo, node, name)
} else {
ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
}
continue
}
var ef protocol.FileInfoTruncated
err = ef.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
// Flags might change without the version being bumped when we set the
// invalid flag on an existing file.
if ef.Version != f.Version || ef.Flags != f.Flags {
if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
maxLocalVer = lv
}
if f.IsInvalid() {
ldbRemoveFromGlobal(snap, batch, repo, node, name)
} else {
ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
}
}
}
err = db.Write(batch, nil)
if err != nil {
panic(err)
}
return maxLocalVer
}
func ldbInsert(batch dbWriter, repo, node, name []byte, file protocol.FileInfo) uint64 {
if debug {
l.Debugf("insert; repo=%q node=%v %v", repo, protocol.NodeIDFromBytes(node), file)
}
if file.LocalVersion == 0 {
file.LocalVersion = clock(0)
}
nk := nodeKey(repo, node, name)
batch.Put(nk, file.MarshalXDR())
return file.LocalVersion
}
// ldbUpdateGlobal adds this node+version to the version list for the given
// file. If the node is already present in the list, the version is updated.
// If the file does not have an entry in the global list, it is created.
func ldbUpdateGlobal(db dbReader, batch dbWriter, repo, node, file []byte, version uint64) bool {
if debug {
l.Debugf("update global; repo=%q node=%v file=%q version=%d", repo, protocol.NodeIDFromBytes(node), file, version)
}
gk := globalKey(repo, file)
svl, err := db.Get(gk, nil)
if err != nil && err != leveldb.ErrNotFound {
panic(err)
}
var fl versionList
nv := fileVersion{
node: node,
version: version,
}
if svl != nil {
err = fl.UnmarshalXDR(svl)
if err != nil {
panic(err)
}
for i := range fl.versions {
if bytes.Compare(fl.versions[i].node, node) == 0 {
if fl.versions[i].version == version {
// No need to do anything
return false
}
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
break
}
}
}
for i := range fl.versions {
if fl.versions[i].version <= version {
t := append(fl.versions, fileVersion{})
copy(t[i+1:], t[i:])
t[i] = nv
fl.versions = t
goto done
}
}
fl.versions = append(fl.versions, nv)
done:
batch.Put(gk, fl.MarshalXDR())
return true
}
// ldbRemoveFromGlobal removes the node from the global version list for the
// given file. If the version list is empty after this, the file entry is
// removed entirely.
func ldbRemoveFromGlobal(db dbReader, batch dbWriter, repo, node, file []byte) {
if debug {
l.Debugf("remove from global; repo=%q node=%v file=%q", repo, protocol.NodeIDFromBytes(node), file)
}
gk := globalKey(repo, file)
svl, err := db.Get(gk, nil)
if err != nil {
// We might be called to "remove" a global version that doesn't exist
// if the first update for the file is already marked invalid.
return
}
var fl versionList
err = fl.UnmarshalXDR(svl)
if err != nil {
panic(err)
}
for i := range fl.versions {
if bytes.Compare(fl.versions[i].node, node) == 0 {
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
break
}
}
if len(fl.versions) == 0 {
batch.Delete(gk)
} else {
batch.Put(gk, fl.MarshalXDR())
}
}
func ldbWithHave(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
start := nodeKey(repo, node, nil) // before all repo/node files
limit := nodeKey(repo, node, []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
for dbi.Next() {
f, err := unmarshalTrunc(dbi.Value(), truncate)
if err != nil {
panic(err)
}
if cont := fn(f); !cont {
return
}
}
}
func ldbWithAllRepoTruncated(db *leveldb.DB, repo []byte, fn func(node []byte, f protocol.FileInfoTruncated) bool) {
runtime.GC()
start := nodeKey(repo, nil, nil) // before all repo/node files
limit := nodeKey(repo, protocol.LocalNodeID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all repo/node files
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
for dbi.Next() {
node := nodeKeyNode(dbi.Key())
var f protocol.FileInfoTruncated
err := f.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
}
if cont := fn(node, f); !cont {
return
}
}
}
func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
nk := nodeKey(repo, node, file)
bs, err := db.Get(nk, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}
}
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f
}
func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo {
k := globalKey(repo, file)
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
bs, err := snap.Get(k, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}
}
if err != nil {
panic(err)
}
var vl versionList
err = vl.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
l.Debugln(k)
panic("no versions?")
}
k = nodeKey(repo, vl.versions[0].node, file)
bs, err = snap.Get(k, nil)
if err != nil {
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
return f
}
func ldbWithGlobal(db *leveldb.DB, repo []byte, truncate bool, fn fileIterator) {
runtime.GC()
start := globalKey(repo, nil)
limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
for dbi.Next() {
var vl versionList
err := vl.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
fk := nodeKey(repo, vl.versions[0].node, globalKeyName(dbi.Key()))
bs, err := snap.Get(fk, nil)
if err != nil {
panic(err)
}
f, err := unmarshalTrunc(bs, truncate)
if err != nil {
panic(err)
}
if cont := fn(f); !cont {
return
}
}
}
func ldbAvailability(db *leveldb.DB, repo, file []byte) []protocol.NodeID {
k := globalKey(repo, file)
bs, err := db.Get(k, nil)
if err == leveldb.ErrNotFound {
return nil
}
if err != nil {
panic(err)
}
var vl versionList
err = vl.UnmarshalXDR(bs)
if err != nil {
panic(err)
}
var nodes []protocol.NodeID
for _, v := range vl.versions {
if v.version != vl.versions[0].version {
break
}
n := protocol.NodeIDFromBytes(v.node)
nodes = append(nodes, n)
}
return nodes
}
func ldbWithNeed(db *leveldb.DB, repo, node []byte, truncate bool, fn fileIterator) {
runtime.GC()
start := globalKey(repo, nil)
limit := globalKey(repo, []byte{0xff, 0xff, 0xff, 0xff})
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
outer:
for dbi.Next() {
var vl versionList
err := vl.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
}
if len(vl.versions) == 0 {
l.Debugln(dbi.Key())
panic("no versions?")
}
have := false // If we have the file, any version
need := false // If we have a lower version of the file
var haveVersion uint64
for _, v := range vl.versions {
if bytes.Compare(v.node, node) == 0 {
have = true
haveVersion = v.version
need = v.version < vl.versions[0].version
break
}
}
if need || !have {
name := globalKeyName(dbi.Key())
needVersion := vl.versions[0].version
inner:
for i := range vl.versions {
if vl.versions[i].version != needVersion {
// We haven't found a valid copy of the file with the needed version.
continue outer
}
fk := nodeKey(repo, vl.versions[i].node, name)
bs, err := snap.Get(fk, nil)
if err != nil {
panic(err)
}
gf, err := unmarshalTrunc(bs, truncate)
if err != nil {
panic(err)
}
if gf.IsInvalid() {
// The file is marked invalid for whatever reason, don't use it.
continue inner
}
if gf.IsDeleted() && !have {
// We don't need deleted files that we don't have
continue outer
}
if debug {
l.Debugf("need repo=%q node=%v name=%q need=%v have=%v haveV=%d globalV=%d", repo, protocol.NodeIDFromBytes(node), name, need, have, haveVersion, vl.versions[0].version)
}
if cont := fn(gf); !cont {
return
}
// This file is handled, no need to look further in the version list
continue outer
}
}
}
}
func ldbListRepos(db *leveldb.DB) []string {
runtime.GC()
start := []byte{keyTypeGlobal}
limit := []byte{keyTypeGlobal + 1}
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
defer dbi.Release()
repoExists := make(map[string]bool)
for dbi.Next() {
repo := string(globalKeyRepo(dbi.Key()))
if !repoExists[repo] {
repoExists[repo] = true
}
}
repos := make([]string, 0, len(repoExists))
for k := range repoExists {
repos = append(repos, k)
}
sort.Strings(repos)
return repos
}
func ldbDropRepo(db *leveldb.DB, repo []byte) {
runtime.GC()
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
defer snap.Release()
// Remove all items related to the given repo from the node->file bucket
start := []byte{keyTypeNode}
limit := []byte{keyTypeNode + 1}
dbi := snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
for dbi.Next() {
itemRepo := nodeKeyRepo(dbi.Key())
if bytes.Compare(repo, itemRepo) == 0 {
db.Delete(dbi.Key(), nil)
}
}
dbi.Release()
// Remove all items related to the given repo from the global bucket
start = []byte{keyTypeGlobal}
limit = []byte{keyTypeGlobal + 1}
dbi = snap.NewIterator(&util.Range{Start: start, Limit: limit}, nil)
for dbi.Next() {
itemRepo := globalKeyRepo(dbi.Key())
if bytes.Compare(repo, itemRepo) == 0 {
db.Delete(dbi.Key(), nil)
}
}
dbi.Release()
}
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
if truncate {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(bs)
return tf, err
} else {
var tf protocol.FileInfo
err := tf.UnmarshalXDR(bs)
return tf, err
}
}

145
internal/files/leveldb_xdr.go Normal file
View File

@@ -0,0 +1,145 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package files
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
/*
fileVersion Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of node |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ node (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct fileVersion {
unsigned hyper version;
opaque node<>;
}
*/
func (o fileVersion) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o fileVersion) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o fileVersion) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o fileVersion) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteUint64(o.version)
xw.WriteBytes(o.node)
return xw.Tot(), xw.Error()
}
func (o *fileVersion) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *fileVersion) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *fileVersion) decodeXDR(xr *xdr.Reader) error {
o.version = xr.ReadUint64()
o.node = xr.ReadBytes()
return xr.Error()
}
/*
versionList Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Number of versions |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Zero or more fileVersion Structures \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct versionList {
fileVersion versions<>;
}
*/
func (o versionList) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o versionList) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o versionList) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o versionList) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteUint32(uint32(len(o.versions)))
for i := range o.versions {
_, err := o.versions[i].encodeXDR(xw)
if err != nil {
return xw.Tot(), err
}
}
return xw.Tot(), xw.Error()
}
func (o *versionList) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *versionList) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *versionList) decodeXDR(xr *xdr.Reader) error {
_versionsSize := int(xr.ReadUint32())
o.versions = make([]fileVersion, _versionsSize)
for i := range o.versions {
(&o.versions[i]).decodeXDR(xr)
}
return xr.Error()
}

188
internal/files/set.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// Package files provides a set type to track local/remote files with newness
// checks. We must do a certain amount of normalization in here. We will get
// fed paths with either native or wire-format separators and encodings
// depending on who calls us. We transform paths to wire-format (NFC and
// slashes) on the way to the database, and transform to native format
// (varying separator and encoding) on the way back out.
package files
import (
"sync"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syndtr/goleveldb/leveldb"
)
type fileRecord struct {
File protocol.FileInfo
Usage int
Global bool
}
type bitset uint64
type Set struct {
localVersion map[protocol.NodeID]uint64
mutex sync.Mutex
repo string
db *leveldb.DB
}
func NewSet(repo string, db *leveldb.DB) *Set {
var s = Set{
localVersion: make(map[protocol.NodeID]uint64),
repo: repo,
db: db,
}
var nodeID protocol.NodeID
ldbWithAllRepoTruncated(db, []byte(repo), func(node []byte, f protocol.FileInfoTruncated) bool {
copy(nodeID[:], node)
if f.LocalVersion > s.localVersion[nodeID] {
s.localVersion[nodeID] = f.LocalVersion
}
lamport.Default.Tick(f.Version)
return true
})
if debug {
l.Debugf("loaded localVersion for %q: %#v", repo, s.localVersion)
}
clock(s.localVersion[protocol.LocalNodeID])
return &s
}
func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) {
if debug {
l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
}
normalizeFilenames(fs)
s.mutex.Lock()
defer s.mutex.Unlock()
s.localVersion[node] = ldbReplace(s.db, []byte(s.repo), node[:], fs)
}
func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) {
if debug {
l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
}
normalizeFilenames(fs)
s.mutex.Lock()
defer s.mutex.Unlock()
if lv := ldbReplaceWithDelete(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
s.localVersion[node] = lv
}
}
func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) {
if debug {
l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
}
normalizeFilenames(fs)
s.mutex.Lock()
defer s.mutex.Unlock()
if lv := ldbUpdate(s.db, []byte(s.repo), node[:], fs); lv > s.localVersion[node] {
s.localVersion[node] = lv
}
}
func (s *Set) WithNeed(node protocol.NodeID, fn fileIterator) {
if debug {
l.Debugf("%s WithNeed(%v)", s.repo, node)
}
ldbWithNeed(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
}
func (s *Set) WithNeedTruncated(node protocol.NodeID, fn fileIterator) {
if debug {
l.Debugf("%s WithNeedTruncated(%v)", s.repo, node)
}
ldbWithNeed(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
}
func (s *Set) WithHave(node protocol.NodeID, fn fileIterator) {
if debug {
l.Debugf("%s WithHave(%v)", s.repo, node)
}
ldbWithHave(s.db, []byte(s.repo), node[:], false, nativeFileIterator(fn))
}
func (s *Set) WithHaveTruncated(node protocol.NodeID, fn fileIterator) {
if debug {
l.Debugf("%s WithHaveTruncated(%v)", s.repo, node)
}
ldbWithHave(s.db, []byte(s.repo), node[:], true, nativeFileIterator(fn))
}
func (s *Set) WithGlobal(fn fileIterator) {
if debug {
l.Debugf("%s WithGlobal()", s.repo)
}
ldbWithGlobal(s.db, []byte(s.repo), false, nativeFileIterator(fn))
}
func (s *Set) WithGlobalTruncated(fn fileIterator) {
if debug {
l.Debugf("%s WithGlobalTruncated()", s.repo)
}
ldbWithGlobal(s.db, []byte(s.repo), true, nativeFileIterator(fn))
}
func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo {
f := ldbGet(s.db, []byte(s.repo), node[:], []byte(normalizedFilename(file)))
f.Name = nativeFilename(f.Name)
return f
}
func (s *Set) GetGlobal(file string) protocol.FileInfo {
f := ldbGetGlobal(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
f.Name = nativeFilename(f.Name)
return f
}
func (s *Set) Availability(file string) []protocol.NodeID {
return ldbAvailability(s.db, []byte(s.repo), []byte(normalizedFilename(file)))
}
func (s *Set) LocalVersion(node protocol.NodeID) uint64 {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.localVersion[node]
}
// ListRepos returns the repository IDs seen in the database.
func ListRepos(db *leveldb.DB) []string {
return ldbListRepos(db)
}
// DropRepo clears out all information related to the given repo from the
// database.
func DropRepo(db *leveldb.DB, repo string) {
ldbDropRepo(db, []byte(repo))
}
func normalizeFilenames(fs []protocol.FileInfo) {
for i := range fs {
fs[i].Name = normalizedFilename(fs[i].Name)
}
}
func nativeFileIterator(fn fileIterator) fileIterator {
return func(fi protocol.FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = nativeFilename(f.Name)
return fn(f)
case protocol.FileInfoTruncated:
f.Name = nativeFilename(f.Name)
return fn(f)
default:
panic("unknown interface type")
}
}
}

913
internal/files/set_test.go Normal file
View File

@@ -0,0 +1,913 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package files_test
import (
"bytes"
"fmt"
"reflect"
"sort"
"testing"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
)
var remoteNode0, remoteNode1 protocol.NodeID
func init() {
remoteNode0, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
remoteNode1, _ = protocol.NodeIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
}
func genBlocks(n int) []protocol.BlockInfo {
b := make([]protocol.BlockInfo, n)
for i := range b {
h := make([]byte, 32)
for j := range h {
h[j] = byte(i + j)
}
b[i].Size = uint32(i)
b[i].Hash = h
}
return b
}
func globalList(s *files.Set) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithGlobal(func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
})
return fs
}
func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithHave(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
})
return fs
}
func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithNeed(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
})
return fs
}
type fileList []protocol.FileInfo
func (l fileList) Len() int {
return len(l)
}
func (l fileList) Less(a, b int) bool {
return l[a].Name < l[b].Name
}
func (l fileList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func (l fileList) String() string {
var b bytes.Buffer
b.WriteString("[]protocol.FileList{\n")
for _, f := range l {
fmt.Fprintf(&b, " %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
}
b.WriteString("}")
return b.String()
}
func TestGlobalSet(t *testing.T) {
lamport.Default = lamport.Clock{}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
local0 := fileList{
protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)},
}
local1 := fileList{
protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
}
localTot := fileList{
local0[0],
local0[1],
local0[2],
local0[3],
protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
}
remote0 := fileList{
protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)},
}
remote1 := fileList{
protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)},
}
remoteTot := fileList{
remote0[0],
remote1[0],
remote0[2],
remote1[1],
}
expectedGlobal := fileList{
remote0[0], // a
remote1[0], // b
remote0[2], // c
localTot[3], // d
remote1[1], // e
localTot[4], // z
}
expectedLocalNeed := fileList{
remote1[0],
remote0[2],
remote1[1],
}
expectedRemoteNeed := fileList{
local0[3],
}
m.ReplaceWithDelete(protocol.LocalNodeID, local0)
m.ReplaceWithDelete(protocol.LocalNodeID, local1)
m.Replace(remoteNode0, remote0)
m.Update(remoteNode0, remote1)
g := fileList(globalList(m))
sort.Sort(g)
if fmt.Sprint(g) != fmt.Sprint(expectedGlobal) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
}
h := fileList(haveList(m, protocol.LocalNodeID))
sort.Sort(h)
if fmt.Sprint(h) != fmt.Sprint(localTot) {
t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
}
h = fileList(haveList(m, remoteNode0))
sort.Sort(h)
if fmt.Sprint(h) != fmt.Sprint(remoteTot) {
t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, remoteTot)
}
n := fileList(needList(m, protocol.LocalNodeID))
sort.Sort(n)
if fmt.Sprint(n) != fmt.Sprint(expectedLocalNeed) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedLocalNeed)
}
n = fileList(needList(m, remoteNode0))
sort.Sort(n)
if fmt.Sprint(n) != fmt.Sprint(expectedRemoteNeed) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
}
f := m.Get(protocol.LocalNodeID, "b")
if fmt.Sprint(f) != fmt.Sprint(localTot[1]) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
}
f = m.Get(remoteNode0, "b")
if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
}
f = m.GetGlobal("b")
if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
}
f = m.Get(protocol.LocalNodeID, "zz")
if f.Name != "" {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
}
f = m.GetGlobal("zz")
if f.Name != "" {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
}
av := []protocol.NodeID{protocol.LocalNodeID, remoteNode0}
a := m.Availability("a")
if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
}
a = m.Availability("b")
if len(a) != 1 || a[0] != remoteNode0 {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteNode0)
}
a = m.Availability("d")
if len(a) != 1 || a[0] != protocol.LocalNodeID {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalNodeID)
}
}
func TestNeedWithInvalid(t *testing.T) {
lamport.Default = lamport.Clock{}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s := files.NewSet("test", db)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
}
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
}
remote1Have := fileList{
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "e", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
}
expectedNeed := fileList{
protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
}
s.ReplaceWithDelete(protocol.LocalNodeID, localHave)
s.Replace(remoteNode0, remote0Have)
s.Replace(remoteNode1, remote1Have)
need := fileList(needList(s, protocol.LocalNodeID))
sort.Sort(need)
if fmt.Sprint(need) != fmt.Sprint(expectedNeed) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed)
}
}
func TestUpdateToInvalid(t *testing.T) {
lamport.Default = lamport.Clock{}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s := files.NewSet("test", db)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
}
s.ReplaceWithDelete(protocol.LocalNodeID, localHave)
have := fileList(haveList(s, protocol.LocalNodeID))
sort.Sort(have)
if fmt.Sprint(have) != fmt.Sprint(localHave) {
t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
}
localHave[1] = protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagInvalid}
s.Update(protocol.LocalNodeID, localHave[1:2])
have = fileList(haveList(s, protocol.LocalNodeID))
sort.Sort(have)
if fmt.Sprint(have) != fmt.Sprint(localHave) {
t.Errorf("Have incorrect after invalidation;\n A: %v !=\n E: %v", have, localHave)
}
}
func TestInvalidAvailability(t *testing.T) {
lamport.Default = lamport.Clock{}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s := files.NewSet("test", db)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
}
remote1Have := fileList{
protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
}
s.Replace(remoteNode0, remote0Have)
s.Replace(remoteNode1, remote1Have)
if av := s.Availability("both"); len(av) != 2 {
t.Error("Incorrect availability for 'both':", av)
}
if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteNode0 {
t.Error("Incorrect availability for 'r0only':", av)
}
if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteNode1 {
t.Error("Incorrect availability for 'r1only':", av)
}
if av := s.Availability("none"); len(av) != 0 {
t.Error("Incorrect availability for 'none':", av)
}
}
func TestLocalDeleted(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
lamport.Default = lamport.Clock{}
local1 := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
protocol.FileInfo{Name: "c", Version: 1000},
protocol.FileInfo{Name: "d", Version: 1000},
protocol.FileInfo{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
}
m.ReplaceWithDelete(protocol.LocalNodeID, local1)
m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0],
// [1] removed
local1[2],
local1[3],
local1[4],
})
m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0],
local1[2],
// [3] removed
local1[4],
})
m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0],
local1[2],
// [4] removed
})
expectedGlobal1 := []protocol.FileInfo{
local1[0],
protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
local1[2],
protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
}
g := globalList(m)
sort.Sort(fileList(g))
sort.Sort(fileList(expectedGlobal1))
if fmt.Sprint(g) != fmt.Sprint(expectedGlobal1) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
}
m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0],
// [2] removed
})
expectedGlobal2 := []protocol.FileInfo{
local1[0],
protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
}
g = globalList(m)
sort.Sort(fileList(g))
sort.Sort(fileList(expectedGlobal2))
if fmt.Sprint(g) != fmt.Sprint(expectedGlobal2) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal2)
}
}
func Benchmark10kReplace(b *testing.B) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
var local []protocol.FileInfo
for i := 0; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
m := files.NewSet("test", db)
m.ReplaceWithDelete(protocol.LocalNodeID, local)
}
}
func Benchmark10kUpdateChg(b *testing.B) {
var remote []protocol.FileInfo
for i := 0; i < 10000; i++ {
remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
m := files.NewSet("test", db)
m.Replace(remoteNode0, remote)
var local []protocol.FileInfo
for i := 0; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
for j := range local {
local[j].Version++
}
b.StartTimer()
m.Update(protocol.LocalNodeID, local)
}
}
func Benchmark10kUpdateSme(b *testing.B) {
var remote []protocol.FileInfo
for i := 0; i < 10000; i++ {
remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
m := files.NewSet("test", db)
m.Replace(remoteNode0, remote)
var local []protocol.FileInfo
for i := 0; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.Update(protocol.LocalNodeID, local)
}
}
func Benchmark10kNeed2k(b *testing.B) {
var remote []protocol.FileInfo
for i := 0; i < 10000; i++ {
remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
m := files.NewSet("test", db)
m.Replace(remoteNode0, remote)
var local []protocol.FileInfo
for i := 0; i < 8000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
for i := 8000; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
fs := needList(m, protocol.LocalNodeID)
if l := len(fs); l != 2000 {
b.Errorf("wrong length %d != 2k", l)
}
}
}
func Benchmark10kHaveFullList(b *testing.B) {
var remote []protocol.FileInfo
for i := 0; i < 10000; i++ {
remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
m := files.NewSet("test", db)
m.Replace(remoteNode0, remote)
var local []protocol.FileInfo
for i := 0; i < 2000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
for i := 2000; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
fs := haveList(m, protocol.LocalNodeID)
if l := len(fs); l != 10000 {
b.Errorf("wrong length %d != 10k", l)
}
}
}
func Benchmark10kGlobal(b *testing.B) {
var remote []protocol.FileInfo
for i := 0; i < 10000; i++ {
remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
b.Fatal(err)
}
m := files.NewSet("test", db)
m.Replace(remoteNode0, remote)
var local []protocol.FileInfo
for i := 0; i < 2000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
}
for i := 2000; i < 10000; i++ {
local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
fs := globalList(m)
if l := len(fs); l != 10000 {
b.Errorf("wrong length %d != 10k", l)
}
}
}
func TestGlobalReset(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
local := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
protocol.FileInfo{Name: "c", Version: 1000},
protocol.FileInfo{Name: "d", Version: 1000},
}
remote := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1001},
protocol.FileInfo{Name: "c", Version: 1002},
protocol.FileInfo{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
g := globalList(m)
sort.Sort(fileList(g))
if fmt.Sprint(g) != fmt.Sprint(local) {
t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
}
m.Replace(remoteNode0, remote)
m.Replace(remoteNode0, nil)
g = globalList(m)
sort.Sort(fileList(g))
if fmt.Sprint(g) != fmt.Sprint(local) {
t.Errorf("Global incorrect;\n%v !=\n%v", g, local)
}
}
func TestNeed(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
local := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
protocol.FileInfo{Name: "c", Version: 1000},
protocol.FileInfo{Name: "d", Version: 1000},
}
remote := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1001},
protocol.FileInfo{Name: "c", Version: 1002},
protocol.FileInfo{Name: "e", Version: 1000},
}
shouldNeed := []protocol.FileInfo{
protocol.FileInfo{Name: "b", Version: 1001},
protocol.FileInfo{Name: "c", Version: 1002},
protocol.FileInfo{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(protocol.LocalNodeID, local)
m.Replace(remoteNode0, remote)
need := needList(m, protocol.LocalNodeID)
sort.Sort(fileList(need))
sort.Sort(fileList(shouldNeed))
if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
}
}
func TestLocalVersion(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
local1 := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
protocol.FileInfo{Name: "c", Version: 1000},
protocol.FileInfo{Name: "d", Version: 1000},
}
local2 := []protocol.FileInfo{
local1[0],
// [1] deleted
local1[2],
protocol.FileInfo{Name: "d", Version: 1002},
protocol.FileInfo{Name: "e", Version: 1000},
}
m.ReplaceWithDelete(protocol.LocalNodeID, local1)
c0 := m.LocalVersion(protocol.LocalNodeID)
m.ReplaceWithDelete(protocol.LocalNodeID, local2)
c1 := m.LocalVersion(protocol.LocalNodeID)
if !(c1 > c0) {
t.Fatal("Local version number should have incremented")
}
m.ReplaceWithDelete(protocol.LocalNodeID, local2)
c2 := m.LocalVersion(protocol.LocalNodeID)
if c2 != c1 {
t.Fatal("Local version number should be unchanged")
}
}
func TestListDropRepo(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s0 := files.NewSet("test0", db)
local1 := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
protocol.FileInfo{Name: "c", Version: 1000},
}
s0.Replace(protocol.LocalNodeID, local1)
s1 := files.NewSet("test1", db)
local2 := []protocol.FileInfo{
protocol.FileInfo{Name: "d", Version: 1002},
protocol.FileInfo{Name: "e", Version: 1002},
protocol.FileInfo{Name: "f", Version: 1002},
}
s1.Replace(remoteNode0, local2)
// Check that we have both repos and their data is in the global list
expectedRepoList := []string{"test0", "test1"}
if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
}
if l := len(globalList(s0)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s0", l)
}
if l := len(globalList(s1)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s1", l)
}
// Drop one of them and check that it's gone.
files.DropRepo(db, "test1")
expectedRepoList = []string{"test0"}
if actualRepoList := files.ListRepos(db); !reflect.DeepEqual(actualRepoList, expectedRepoList) {
t.Fatalf("RepoList mismatch\nE: %v\nA: %v", expectedRepoList, actualRepoList)
}
if l := len(globalList(s0)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s0", l)
}
if l := len(globalList(s1)); l != 0 {
t.Errorf("Incorrect global length %d != 0 for s1", l)
}
}
func TestGlobalNeedWithInvalid(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s := files.NewSet("test1", db)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: 1002, Flags: protocol.FlagInvalid},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
}
s.Replace(remoteNode0, rem0)
rem1 := fileList{
protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: 1002, Flags: protocol.FlagInvalid},
}
s.Replace(remoteNode1, rem1)
total := fileList{
// There's a valid copy of each file, so it should be merged
protocol.FileInfo{Name: "a", Version: 1002, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: 1002, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(4)},
}
need := fileList(needList(s, protocol.LocalNodeID))
if fmt.Sprint(need) != fmt.Sprint(total) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total)
}
global := fileList(globalList(s))
if fmt.Sprint(global) != fmt.Sprint(total) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", global, total)
}
}
func TestLongPath(t *testing.T) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
t.Fatal(err)
}
s := files.NewSet("test", db)
var b bytes.Buffer
for i := 0; i < 100; i++ {
b.WriteString("012345678901234567890123456789012345678901234567890")
}
name := b.String() // 5000 characters
local := []protocol.FileInfo{
protocol.FileInfo{Name: string(name), Version: 1000},
}
s.ReplaceWithDelete(protocol.LocalNodeID, local)
gf := globalList(s)
if l := len(gf); l != 1 {
t.Fatalf("Incorrect len %d != 1 for global list", l)
}
if gf[0].Name != local[0].Name {
t.Error("Incorrect long filename;\n%q !=\n%q", gf[0].Name, local[0].Name)
}
}
/*
var gf protocol.FileInfo
func TestStressGlobalVersion(t *testing.T) {
dur := 15 * time.Second
if testing.Short() {
dur = 1 * time.Second
}
set1 := []protocol.FileInfo{
protocol.FileInfo{Name: "a", Version: 1000},
protocol.FileInfo{Name: "b", Version: 1000},
}
set2 := []protocol.FileInfo{
protocol.FileInfo{Name: "b", Version: 1001},
protocol.FileInfo{Name: "c", Version: 1000},
}
db, err := leveldb.OpenFile("testdata/global.db", nil)
if err != nil {
t.Fatal(err)
}
m := files.NewSet("test", db)
done := make(chan struct{})
go stressWriter(m, remoteNode0, set1, nil, done)
go stressWriter(m, protocol.LocalNodeID, set2, nil, done)
t0 := time.Now()
for time.Since(t0) < dur {
m.WithGlobal(func(f protocol.FileInfo) bool {
gf = f
return true
})
}
close(done)
}
func stressWriter(s *files.Set, id protocol.NodeID, set1, set2 []protocol.FileInfo, done chan struct{}) {
one := true
i := 0
for {
select {
case <-done:
return
default:
if one {
s.Replace(id, set1)
} else {
s.Replace(id, set2)
}
one = !one
}
i++
}
}
*/

1
internal/files/testdata/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
index.db