perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing, lib/db, lib/model: Track more detailed file/dirs/links/deleted counts

Browse Source
This commit is contained in:
Jakob Borg
2016-10-17 14:10:17 +02:00
parent b8a90b7eaa
commit 4e8c8d7e2c
8 changed files with 143 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
lib/db/set.go
View File

@@ -51,11 +51,17 @@ type FileIntf interface {
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
type Counts struct {
Files int
Directories int
Symlinks int
Deleted int
Bytes int64
}
type sizeTracker struct {
files int
deleted int
bytes int64
mut stdsync.Mutex
Counts
mut stdsync.Mutex
}
func (s *sizeTracker) addFile(f FileIntf) {
@@ -64,12 +70,17 @@ func (s *sizeTracker) addFile(f FileIntf) {
}
s.mut.Lock()
if f.IsDeleted() {
s.deleted++
} else {
s.files++
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
s.bytes += f.FileSize()
s.Bytes += f.FileSize()
s.mut.Unlock()
}
@@ -79,22 +90,27 @@ func (s *sizeTracker) removeFile(f FileIntf) {
}
s.mut.Lock()
if f.IsDeleted() {
s.deleted--
} else {
s.files--
switch {
case f.IsDeleted():
s.Deleted--
case f.IsDirectory():
s.Directories--
case f.IsSymlink():
s.Symlinks--
default:
s.Files--
}
s.bytes -= f.FileSize()
if s.deleted < 0 || s.files < 0 {
s.Bytes -= f.FileSize()
if s.Deleted < 0 || s.Files < 0 || s.Directories < 0 || s.Symlinks < 0 {
panic("bug: removed more than added")
}
s.mut.Unlock()
}
func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
func (s *sizeTracker) Size() Counts {
s.mut.Lock()
defer s.mut.Unlock()
return s.files, s.deleted, s.bytes
return s.Counts
}
func NewFileSet(folder string, db *Instance) *FileSet {
@@ -259,11 +275,11 @@ func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.remoteSequence[device]
}
func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
func (s *FileSet) LocalSize() Counts {
return s.localSize.Size()
}
func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
func (s *FileSet) GlobalSize() Counts {
return s.globalSize.Size()
}

52
lib/db/set_test.go
View File

@@ -168,27 +168,33 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
}
globalFiles, globalDeleted, globalBytes := 0, 0, int64(0)
globalFiles, globalDirectories, globalDeleted, globalBytes := 0, 0, 0, int64(0)
for _, f := range g {
if f.IsInvalid() {
continue
}
if f.IsDeleted() {
switch {
case f.IsDeleted():
globalDeleted++
} else {
case f.IsDirectory():
globalDirectories++
default:
globalFiles++
}
globalBytes += f.FileSize()
}
gsFiles, gsDeleted, gsBytes := m.GlobalSize()
if gsFiles != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gsFiles, globalFiles)
gs := m.GlobalSize()
if gs.Files != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles)
}
if gsDeleted != globalDeleted {
t.Errorf("Incorrect GlobalSize deleted; %d != %d", gsDeleted, globalDeleted)
if gs.Directories != globalDirectories {
t.Errorf("Incorrect GlobalSize directories; %d != %d", gs.Directories, globalDirectories)
}
if gsBytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gsBytes, globalBytes)
if gs.Deleted != globalDeleted {
t.Errorf("Incorrect GlobalSize deleted; %d != %d", gs.Deleted, globalDeleted)
}
if gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
h := fileList(haveList(m, protocol.LocalDeviceID))
@@ -198,27 +204,33 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Have incorrect;\n A: %v !=\n E: %v", h, localTot)
}
haveFiles, haveDeleted, haveBytes := 0, 0, int64(0)
haveFiles, haveDirectories, haveDeleted, haveBytes := 0, 0, 0, int64(0)
for _, f := range h {
if f.IsInvalid() {
continue
}
if f.IsDeleted() {
switch {
case f.IsDeleted():
haveDeleted++
} else {
case f.IsDirectory():
haveDirectories++
default:
haveFiles++
}
haveBytes += f.FileSize()
}
lsFiles, lsDeleted, lsBytes := m.LocalSize()
if lsFiles != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", lsFiles, haveFiles)
ls := m.LocalSize()
if ls.Files != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles)
}
if lsDeleted != haveDeleted {
t.Errorf("Incorrect LocalSize deleted; %d != %d", lsDeleted, haveDeleted)
if ls.Directories != haveDirectories {
t.Errorf("Incorrect LocalSize directories; %d != %d", ls.Directories, haveDirectories)
}
if lsBytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", lsBytes, haveBytes)
if ls.Deleted != haveDeleted {
t.Errorf("Incorrect LocalSize deleted; %d != %d", ls.Deleted, haveDeleted)
}
if ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
h = fileList(haveList(m, remoteDevice0))

46
lib/model/model.go
View File

@@ -476,7 +476,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
}
_, _, tot := rf.GlobalSize()
tot := rf.GlobalSize().Bytes
if tot == 0 {
// Folder is empty, so we have all of it
return FolderCompletion{
@@ -531,42 +531,49 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
}
}
func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() {
files++
} else {
deleted++
func addSizeOfFile(s *db.Counts, f db.FileIntf) {
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
bytes += f.FileSize()
s.Bytes += f.FileSize()
return
}
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
func (m *Model) GlobalSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
nfiles, deleted, bytes = rf.GlobalSize()
return rf.GlobalSize()
}
return
return db.Counts{}
}
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
func (m *Model) LocalSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
nfiles, deleted, bytes = rf.LocalSize()
return rf.LocalSize()
}
return
return db.Counts{}
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (nfiles, ndeletes int, bytes int64) {
func (m *Model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
var result db.Counts
if rf, ok := m.folderFiles[folder]; ok {
ignores := m.folderIgnores[folder]
cfg := m.folderCfgs[folder]
@@ -575,16 +582,13 @@ func (m *Model) NeedSize(folder string) (nfiles, ndeletes int, bytes int64) {
return true
}
fs, de, by := sizeOfFile(f)
nfiles += fs
ndeletes += de
bytes += by
addSizeOfFile(&result, f)
return true
})
}
bytes -= m.progressEmitter.BytesCompleted(folder)
l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
return
result.Bytes -= m.progressEmitter.BytesCompleted(folder)
l.Debugf("%v NeedSize(%q): %v", m, folder, result)
return result
}
// NeedFolderFiles returns paginated list of currently needed files in

34
lib/model/model_test.go
View File

@@ -1324,8 +1324,8 @@ func TestIssue3028(t *testing.T) {
// Get a count of how many files are there now
locorigfiles, _, _ := m.LocalSize("default")
globorigfiles, _, _ := m.GlobalSize("default")
locorigfiles := m.LocalSize("default").Files
globorigfiles := m.GlobalSize("default").Files
// Delete and rescan specifically these two
@@ -1336,19 +1336,19 @@ func TestIssue3028(t *testing.T) {
// Verify that the number of files decreased by two and the number of
// deleted files increases by two
locnowfiles, locdelfiles, _ := m.LocalSize("default")
globnowfiles, globdelfiles, _ := m.GlobalSize("default")
if locnowfiles != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", locnowfiles, locorigfiles-2)
loc := m.LocalSize("default")
glob := m.GlobalSize("default")
if loc.Files != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2)
}
if globnowfiles != globorigfiles-2 {
t.Errorf("Incorrect global accounting; got %d current files, expected %d", globnowfiles, globorigfiles-2)
if glob.Files != globorigfiles-2 {
t.Errorf("Incorrect global accounting; got %d current files, expected %d", glob.Files, globorigfiles-2)
}
if locdelfiles != 2 {
t.Errorf("Incorrect local accounting; got %d deleted files, expected 2", locdelfiles)
if loc.Deleted != 2 {
t.Errorf("Incorrect local accounting; got %d deleted files, expected 2", loc.Deleted)
}
if globdelfiles != 2 {
t.Errorf("Incorrect global accounting; got %d deleted files, expected 2", globdelfiles)
if glob.Deleted != 2 {
t.Errorf("Incorrect global accounting; got %d deleted files, expected 2", glob.Deleted)
}
}
@@ -1722,14 +1722,14 @@ func TestIssue3496(t *testing.T) {
t.Log(comp)
// Check that NeedSize does the correct thing
files, deletes, bytes := m.NeedSize("default")
if files != 1 || bytes != 1234 {
need := m.NeedSize("default")
if need.Files != 1 || need.Bytes != 1234 {
// The one we added synthetically above
t.Errorf("Incorrect need size; %d, %d != 1, 1234", files, bytes)
t.Errorf("Incorrect need size; %d, %d != 1, 1234", need.Files, need.Bytes)
}
if deletes != len(localFiles)-1 {
if need.Deleted != len(localFiles)-1 {
// The rest
t.Errorf("Incorrect need deletes; %d != %d", deletes, len(localFiles)-1)
t.Errorf("Incorrect need deletes; %d != %d", need.Deleted, len(localFiles)-1)
}
}