perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move FileIntf to files package, expose Iterator type

Browse Source 
This is where FileIntf is used, so it should be defined here (it's not
a protocol thing, really).
This commit is contained in:
Jakob Borg 2015-01-09 08:18:42 +01:00
parent 2d047fa428
commit 8bc7d259f4
7 changed files with 55 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/stindex/main.go
View File

@ -43,7 +43,7 @@ func main() {
if *device == "" { if *device == "" {
log.Printf("*** Global index for folder %q", *folder) log.Printf("*** Global index for folder %q", *folder)
fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool { fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated) f := fi.(protocol.FileInfoTruncated)
fmt.Println(f) fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name)) fmt.Println("\t", fs.Availability(f.Name))
@ -55,7 +55,7 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("*** Have index for folder %q device %q", *folder, n) log.Printf("*** Have index for folder %q device %q", *folder, n)
fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool { fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated) f := fi.(protocol.FileInfoTruncated)
fmt.Println(f) fmt.Println(f)
return true return true

10
internal/files/leveldb.go
View File

@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
type fileIterator func(f protocol.FileIntf) bool
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 { func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
runtime.GC() runtime.GC()
@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
} }
} }
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) { func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
start := deviceKey(folder, device, nil) // before all folder/device files start := deviceKey(folder, device, nil) // before all folder/device files
limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
snap, err := db.GetSnapshot() snap, err := db.GetSnapshot()
@ -666,7 +664,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
return f, true return f, true
} }
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) { func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
runtime.GC() runtime.GC()
start := globalKey(folder, nil) start := globalKey(folder, nil)
@ -754,7 +752,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
return devices return devices
} }
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) { func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
runtime.GC() runtime.GC()
start := globalKey(folder, nil) start := globalKey(folder, nil)
@ -938,7 +936,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
dbi.Release() dbi.Release()
} }
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) { func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate { if truncate {
var tf protocol.FileInfoTruncated var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(bs) err := tf.UnmarshalXDR(bs)

32
internal/files/set.go
View File

@ -38,6 +38,22 @@ type Set struct {
blockmap *BlockMap blockmap *BlockMap
} }
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
func NewSet(folder string, db *leveldb.DB) *Set { func NewSet(folder string, db *leveldb.DB) *Set {
var s = Set{ var s = Set{
localVersion: make(map[protocol.DeviceID]uint64), localVersion: make(map[protocol.DeviceID]uint64),
@ -124,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
} }
} }
func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) { func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
if debug { if debug {
l.Debugf("%s WithNeed(%v)", s.folder, device) l.Debugf("%s WithNeed(%v)", s.folder, device)
} }
ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn)) ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
} }
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) { func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
if debug { if debug {
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device) l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
} }
ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn)) ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
} }
func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) { func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
if debug { if debug {
l.Debugf("%s WithHave(%v)", s.folder, device) l.Debugf("%s WithHave(%v)", s.folder, device)
} }
ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn)) ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
} }
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) { func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
if debug { if debug {
l.Debugf("%s WithHaveTruncated(%v)", s.folder, device) l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
} }
ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn)) ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
} }
func (s *Set) WithGlobal(fn fileIterator) { func (s *Set) WithGlobal(fn Iterator) {
if debug { if debug {
l.Debugf("%s WithGlobal()", s.folder) l.Debugf("%s WithGlobal()", s.folder)
} }
ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn)) ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
} }
func (s *Set) WithGlobalTruncated(fn fileIterator) { func (s *Set) WithGlobalTruncated(fn Iterator) {
if debug { if debug {
l.Debugf("%s WithGlobalTruncated()", s.folder) l.Debugf("%s WithGlobalTruncated()", s.folder)
} }
@ -210,8 +226,8 @@ func normalizeFilenames(fs []protocol.FileInfo) {
} }
} }
func nativeFileIterator(fn fileIterator) fileIterator { func nativeFileIterator(fn Iterator) Iterator {
return func(fi protocol.FileIntf) bool { return func(fi FileIntf) bool {
switch f := fi.(type) { switch f := fi.(type) {
case protocol.FileInfo: case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name) f.Name = osutil.NativeFilename(f.Name)

6
internal/files/set_test.go
View File

@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
func globalList(s *files.Set) []protocol.FileInfo { func globalList(s *files.Set) []protocol.FileInfo {
var fs []protocol.FileInfo var fs []protocol.FileInfo
s.WithGlobal(func(fi protocol.FileIntf) bool { s.WithGlobal(func(fi FileIntf) bool {
f := fi.(protocol.FileInfo) f := fi.(protocol.FileInfo)
fs = append(fs, f) fs = append(fs, f)
return true return true
@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo { func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo var fs []protocol.FileInfo
s.WithHave(n, func(fi protocol.FileIntf) bool { s.WithHave(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo) f := fi.(protocol.FileInfo)
fs = append(fs, f) fs = append(fs, f)
return true return true
@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo { func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo var fs []protocol.FileInfo
s.WithNeed(n, func(fi protocol.FileIntf) bool { s.WithNeed(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo) f := fi.(protocol.FileInfo)
fs = append(fs, f) fs = append(fs, f)
return true return true

38
internal/model/model.go
View File

@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
return 0 // Folder doesn't exist, so we hardly have any of it return 0 // Folder doesn't exist, so we hardly have any of it
} }
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool { rf.WithGlobalTruncated(func(f files.FileIntf) bool {
if !f.IsDeleted() { if !f.IsDeleted() {
tot += f.Size() tot += f.Size()
} }
@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
} }
var need int64 var need int64
rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool { rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
if !f.IsDeleted() { if !f.IsDeleted() {
need += f.Size() need += f.Size()
} }
@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
return return
} }
func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) { func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() { if !f.IsDeleted() {
files++ files++
} else { } else {
@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
// GlobalSize returns the number of files, deleted files and total bytes for all // GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model. // files in the global model.
func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) { func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround() defer m.leveldbPanicWorkaround()
m.fmut.RLock() m.fmut.RLock()
defer m.fmut.RUnlock() defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok { if rf, ok := m.folderFiles[folder]; ok {
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool { rf.WithGlobalTruncated(func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs nfiles += fs
deleted += de deleted += de
bytes += by bytes += by
return true return true
@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
// LocalSize returns the number of files, deleted files and total bytes for all // LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder. // files in the local folder.
func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) { func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround() defer m.leveldbPanicWorkaround()
m.fmut.RLock() m.fmut.RLock()
defer m.fmut.RUnlock() defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok { if rf, ok := m.folderFiles[folder]; ok {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
if f.IsInvalid() { if f.IsInvalid() {
return true return true
} }
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs nfiles += fs
deleted += de deleted += de
bytes += by bytes += by
return true return true
@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
} }
// NeedSize returns the number and total size of currently needed files. // NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (files int, bytes int64) { func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
defer m.leveldbPanicWorkaround() defer m.leveldbPanicWorkaround()
m.fmut.RLock() m.fmut.RLock()
defer m.fmut.RUnlock() defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok { if rf, ok := m.folderFiles[folder]; ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs + de nfiles += fs + de
bytes += by bytes += by
return true return true
}) })
} }
bytes -= m.progressEmitter.BytesCompleted(folder) bytes -= m.progressEmitter.BytesCompleted(folder)
if debug { if debug {
l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes) l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
} }
return return
} }
@ -441,21 +441,21 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
for i, name := range progressNames { for i, name := range progressNames {
if f, ok := rf.GetGlobal(name); ok { if f, ok := rf.GetGlobal(name); ok {
progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly progress[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true seen[name] = true
} }
} }
for i, name := range queuedNames { for i, name := range queuedNames {
if f, ok := rf.GetGlobal(name); ok { if f, ok := rf.GetGlobal(name); ok {
queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly queued[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true seen[name] = true
} }
} }
} }
left := max - len(progress) - len(queued) left := max - len(progress) - len(queued)
if max < 1 || left > 0 { if max < 1 || left > 0 {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
left-- left--
ft := f.(protocol.FileInfoTruncated) ft := f.(protocol.FileInfoTruncated)
if !seen[ft.Name] { if !seen[ft.Name] {
@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer := uint64(0) maxLocalVer := uint64(0)
var err error var err error
fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo) f := fi.(protocol.FileInfo)
if f.LocalVersion <= minLocalVer { if f.LocalVersion <= minLocalVer {
return true return true
@ -1169,7 +1169,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0] batch = batch[:0]
// TODO: We should limit the Have scanning to start at sub // TODO: We should limit the Have scanning to start at sub
seenPrefix := false seenPrefix := false
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated) f := fi.(protocol.FileInfoTruncated)
if !strings.HasPrefix(f.Name, sub) { if !strings.HasPrefix(f.Name, sub) {
// Return true so that we keep iterating, until we get to the part // Return true so that we keep iterating, until we get to the part
@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
m.setState(folder, FolderScanning) m.setState(folder, FolderScanning)
batch := make([]protocol.FileInfo, 0, indexBatchSize) batch := make([]protocol.FileInfo, 0, indexBatchSize)
fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
need := fi.(protocol.FileInfo) need := fi.(protocol.FileInfo)
if len(batch) == indexBatchSize { if len(batch) == indexBatchSize {
fs.Update(protocol.LocalDeviceID, batch) fs.Update(protocol.LocalDeviceID, batch)

5
internal/model/puller.go
View File

@ -29,6 +29,7 @@ import (
"github.com/syncthing/syncthing/internal/config" "github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/events" "github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/ignore" "github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol" "github.com/syncthing/syncthing/internal/protocol"
@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
} }
p.model.fmut.RLock() p.model.fmut.RLock()
files := p.model.folderFiles[p.folder] folderFiles := p.model.folderFiles[p.folder]
p.model.fmut.RUnlock() p.model.fmut.RUnlock()
// !!! // !!!
@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
var deletions []protocol.FileInfo var deletions []protocol.FileInfo
files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
// Needed items are delivered sorted lexicographically. This isn't // Needed items are delivered sorted lexicographically. This isn't
// really optimal from a performance point of view - it would be // really optimal from a performance point of view - it would be

9
internal/protocol/message.go
View File

@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0 return f.Flags&FlagNoPermBits == 0
} }
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
type BlockInfo struct { type BlockInfo struct {
Offset int64 // noencode (cache only) Offset int64 // noencode (cache only)
Size uint32 Size uint32