diff --git a/cmd/stindex/main.go b/cmd/stindex/main.go index 01715092..49c4f50c 100644 --- a/cmd/stindex/main.go +++ b/cmd/stindex/main.go @@ -43,7 +43,7 @@ func main() { if *device == "" { log.Printf("*** Global index for folder %q", *folder) - fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool { + fs.WithGlobalTruncated(func(fi files.FileIntf) bool { f := fi.(protocol.FileInfoTruncated) fmt.Println(f) fmt.Println("\t", fs.Availability(f.Name)) @@ -55,7 +55,7 @@ func main() { log.Fatal(err) } log.Printf("*** Have index for folder %q device %q", *folder, n) - fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool { + fs.WithHaveTruncated(n, func(fi files.FileIntf) bool { f := fi.(protocol.FileInfoTruncated) fmt.Println(f) return true diff --git a/internal/files/leveldb.go b/internal/files/leveldb.go index c6688431..51c7fb3f 100644 --- a/internal/files/leveldb.go +++ b/internal/files/leveldb.go @@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte { type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 -type fileIterator func(f protocol.FileIntf) bool - func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 { runtime.GC() @@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt } } -func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) { +func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) { start := deviceKey(folder, device, nil) // before all folder/device files limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files snap, err := db.GetSnapshot() @@ -666,7 +664,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool) return f, true } -func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) { +func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) { runtime.GC() start := globalKey(folder, nil) @@ -754,7 +752,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID { return devices } -func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) { +func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) { runtime.GC() start := globalKey(folder, nil) @@ -938,7 +936,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) { dbi.Release() } -func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) { +func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { if truncate { var tf protocol.FileInfoTruncated err := tf.UnmarshalXDR(bs) diff --git a/internal/files/set.go b/internal/files/set.go index 45b6bf62..4c1c418b 100644 --- a/internal/files/set.go +++ b/internal/files/set.go @@ -38,6 +38,22 @@ type Set struct { blockmap *BlockMap } +// FileIntf is the set of methods implemented by both protocol.FileInfo and +// protocol.FileInfoTruncated. +type FileIntf interface { + Size() int64 + IsDeleted() bool + IsInvalid() bool + IsDirectory() bool + IsSymlink() bool + HasPermissionBits() bool +} + +// The Iterator is called with either a protocol.FileInfo or a +// protocol.FileInfoTruncated (depending on the method) and returns true to +// continue iteration, false to stop. +type Iterator func(f FileIntf) bool + func NewSet(folder string, db *leveldb.DB) *Set { var s = Set{ localVersion: make(map[protocol.DeviceID]uint64), @@ -124,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) { } } -func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) { +func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) { if debug { l.Debugf("%s WithNeed(%v)", s.folder, device) } ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn)) } -func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) { +func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) { if debug { l.Debugf("%s WithNeedTruncated(%v)", s.folder, device) } ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn)) } -func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) { +func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) { if debug { l.Debugf("%s WithHave(%v)", s.folder, device) } ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn)) } -func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) { +func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) { if debug { l.Debugf("%s WithHaveTruncated(%v)", s.folder, device) } ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn)) } -func (s *Set) WithGlobal(fn fileIterator) { +func (s *Set) WithGlobal(fn Iterator) { if debug { l.Debugf("%s WithGlobal()", s.folder) } ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn)) } -func (s *Set) WithGlobalTruncated(fn fileIterator) { +func (s *Set) WithGlobalTruncated(fn Iterator) { if debug { l.Debugf("%s WithGlobalTruncated()", s.folder) } @@ -210,8 +226,8 @@ func normalizeFilenames(fs []protocol.FileInfo) { } } -func nativeFileIterator(fn fileIterator) fileIterator { - return func(fi protocol.FileIntf) bool { +func nativeFileIterator(fn Iterator) Iterator { + return func(fi FileIntf) bool { switch f := fi.(type) { case protocol.FileInfo: f.Name = osutil.NativeFilename(f.Name) diff --git a/internal/files/set_test.go b/internal/files/set_test.go index ed503e02..44839fc5 100644 --- a/internal/files/set_test.go +++ b/internal/files/set_test.go @@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo { func globalList(s *files.Set) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithGlobal(func(fi protocol.FileIntf) bool { + s.WithGlobal(func(fi FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo { func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithHave(n, func(fi protocol.FileIntf) bool { + s.WithHave(n, func(fi FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo { func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithNeed(n, func(fi protocol.FileIntf) bool { + s.WithNeed(n, func(fi FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true diff --git a/internal/model/model.go b/internal/model/model.go index 5aaf6ba2..b598d57e 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 { return 0 // Folder doesn't exist, so we hardly have any of it } - rf.WithGlobalTruncated(func(f protocol.FileIntf) bool { + rf.WithGlobalTruncated(func(f files.FileIntf) bool { if !f.IsDeleted() { tot += f.Size() } @@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 { } var need int64 - rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool { + rf.WithNeedTruncated(device, func(f files.FileIntf) bool { if !f.IsDeleted() { need += f.Size() } @@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) { return } -func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) { +func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) { if !f.IsDeleted() { files++ } else { @@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) { // GlobalSize returns the number of files, deleted files and total bytes for all // files in the global model. -func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) { +func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) { defer m.leveldbPanicWorkaround() m.fmut.RLock() defer m.fmut.RUnlock() if rf, ok := m.folderFiles[folder]; ok { - rf.WithGlobalTruncated(func(f protocol.FileIntf) bool { + rf.WithGlobalTruncated(func(f files.FileIntf) bool { fs, de, by := sizeOfFile(f) - files += fs + nfiles += fs deleted += de bytes += by return true @@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) { // LocalSize returns the number of files, deleted files and total bytes for all // files in the local folder. -func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) { +func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) { defer m.leveldbPanicWorkaround() m.fmut.RLock() defer m.fmut.RUnlock() if rf, ok := m.folderFiles[folder]; ok { - rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { + rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool { if f.IsInvalid() { return true } fs, de, by := sizeOfFile(f) - files += fs + nfiles += fs deleted += de bytes += by return true @@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) { } // NeedSize returns the number and total size of currently needed files. -func (m *Model) NeedSize(folder string) (files int, bytes int64) { +func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) { defer m.leveldbPanicWorkaround() m.fmut.RLock() defer m.fmut.RUnlock() if rf, ok := m.folderFiles[folder]; ok { - rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { + rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool { fs, de, by := sizeOfFile(f) - files += fs + de + nfiles += fs + de bytes += by return true }) } bytes -= m.progressEmitter.BytesCompleted(folder) if debug { - l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes) + l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes) } return } @@ -441,21 +441,21 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun for i, name := range progressNames { if f, ok := rf.GetGlobal(name); ok { - progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly + progress[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly seen[name] = true } } for i, name := range queuedNames { if f, ok := rf.GetGlobal(name); ok { - queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly + queued[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly seen[name] = true } } } left := max - len(progress) - len(queued) if max < 1 || left > 0 { - rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { + rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool { left-- ft := f.(protocol.FileInfoTruncated) if !seen[ft.Name] { @@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol maxLocalVer := uint64(0) var err error - fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { + fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool { f := fi.(protocol.FileInfo) if f.LocalVersion <= minLocalVer { return true @@ -1169,7 +1169,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error { batch = batch[:0] // TODO: We should limit the Have scanning to start at sub seenPrefix := false - fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { + fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool { f := fi.(protocol.FileInfoTruncated) if !strings.HasPrefix(f.Name, sub) { // Return true so that we keep iterating, until we get to the part @@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) { m.setState(folder, FolderScanning) batch := make([]protocol.FileInfo, 0, indexBatchSize) - fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { + fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool { need := fi.(protocol.FileInfo) if len(batch) == indexBatchSize { fs.Update(protocol.LocalDeviceID, batch) diff --git a/internal/model/puller.go b/internal/model/puller.go index 3cffc1e0..0070d10c 100644 --- a/internal/model/puller.go +++ b/internal/model/puller.go @@ -29,6 +29,7 @@ import ( "github.com/syncthing/syncthing/internal/config" "github.com/syncthing/syncthing/internal/events" + "github.com/syncthing/syncthing/internal/files" "github.com/syncthing/syncthing/internal/ignore" "github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/protocol" @@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int { } p.model.fmut.RLock() - files := p.model.folderFiles[p.folder] + folderFiles := p.model.folderFiles[p.folder] p.model.fmut.RUnlock() // !!! @@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int { var deletions []protocol.FileInfo - files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { + folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool { // Needed items are delivered sorted lexicographically. This isn't // really optimal from a performance point of view - it would be diff --git a/internal/protocol/message.go b/internal/protocol/message.go index e0c66b9d..3129f3de 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } -type FileIntf interface { - Size() int64 - IsDeleted() bool - IsInvalid() bool - IsDirectory() bool - IsSymlink() bool - HasPermissionBits() bool -} - type BlockInfo struct { Offset int64 // noencode (cache only) Size uint32