diff --git a/lib/model/folder.go b/lib/model/folder.go index 69895bca..0a155041 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -198,9 +198,9 @@ func (f *folder) Serve() { func (f *folder) BringToFront(string) {} -func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} +func (f *folder) Override() {} -func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} +func (f *folder) Revert() {} func (f *folder) DelayScan(next time.Duration) { f.Delay(next) @@ -345,7 +345,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { Subs: subDirs, Matcher: f.ignores, TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour, - CurrentFiler: cFiler{f.model, f.ID}, + CurrentFiler: cFiler{f.fset}, Filesystem: mtimefs, IgnorePerms: f.IgnorePerms, AutoNormalize: f.AutoNormalize, @@ -361,7 +361,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err) return err } - f.model.updateLocalsFromScanning(f.ID, fs) + f.updateLocalsFromScanning(fs) return nil } // Resolve items which are identical with the global state. @@ -737,6 +737,86 @@ func (f *folder) Errors() []FileError { return append([]FileError{}, f.scanErrors...) } +// ForceRescan marks the file such that it gets rehashed on next scan and then +// immediately executes that scan. +func (f *folder) ForceRescan(file protocol.FileInfo) error { + file.SetMustRescan(f.shortID) + f.fset.Update(protocol.LocalDeviceID, []protocol.FileInfo{file}) + + return f.Scan([]string{file.Name}) +} + +func (f *folder) updateLocalsFromScanning(fs []protocol.FileInfo) { + f.updateLocals(fs) + + f.emitDiskChangeEvents(fs, events.LocalChangeDetected) +} + +func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) { + f.updateLocals(fs) + + f.emitDiskChangeEvents(fs, events.RemoteChangeDetected) +} + +func (f *folder) updateLocals(fs []protocol.FileInfo) { + f.fset.Update(protocol.LocalDeviceID, fs) + + filenames := make([]string, len(fs)) + for i, file := range fs { + filenames[i] = file.Name + } + + events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{ + "folder": f.ID, + "items": len(fs), + "filenames": filenames, + "version": f.fset.Sequence(protocol.LocalDeviceID), + }) +} + +func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events.EventType) { + for _, file := range fs { + if file.IsInvalid() { + continue + } + + objType := "file" + action := "modified" + + switch { + case file.IsDeleted(): + action = "deleted" + + // If our local vector is version 1 AND it is the only version + // vector so far seen for this file then it is a new file. Else if + // it is > 1 it's not new, and if it is 1 but another shortId + // version vector exists then it is new for us but created elsewhere + // so the file is still not new but modified by us. Only if it is + // truly new do we change this to 'added', else we leave it as + // 'modified'. + case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1: + action = "added" + } + + if file.IsSymlink() { + objType = "symlink" + } else if file.IsDirectory() { + objType = "dir" + } + + // Two different events can be fired here based on what EventType is passed into function + events.Default.Log(typeOfEvent, map[string]string{ + "folder": f.ID, + "folderID": f.ID, // incorrect, deprecated, kept for historical compliance + "label": f.Label, + "action": action, + "type": objType, + "path": filepath.FromSlash(file.Name), + "modifiedBy": file.ModifiedBy.String(), + }) + } +} + // The exists function is expected to return true for all known paths // (excluding "" and ".") func unifySubs(dirs []string, exists func(dir string) bool) []string { @@ -770,3 +850,12 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string { } return dirs } + +type cFiler struct { + *db.FileSet +} + +// Implements scanner.CurrentFiler +func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) { + return cf.Get(protocol.LocalDeviceID, file) +} diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go index 6ba0c984..082a0b66 100644 --- a/lib/model/folder_recvonly.go +++ b/lib/model/folder_recvonly.go @@ -62,7 +62,7 @@ func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matche return &receiveOnlyFolder{sr} } -func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) { +func (f *receiveOnlyFolder) Revert() { f.setState(FolderScanning) defer f.setState(FolderIdle) @@ -78,7 +78,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) batchSizeBytes := 0 - fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { fi := intf.(protocol.FileInfo) if !fi.IsReceiveOnlyChanged() { // We're only interested in files that have changed locally in @@ -124,14 +124,14 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File batchSizeBytes += fi.ProtoSize() if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes { - updateFn(batch) + f.updateLocalsFromScanning(batch) batch = batch[:0] batchSizeBytes = 0 } return true }) if len(batch) > 0 { - updateFn(batch) + f.updateLocalsFromScanning(batch) } batch = batch[:0] batchSizeBytes = 0 @@ -153,7 +153,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File }) } if len(batch) > 0 { - updateFn(batch) + f.updateLocalsFromScanning(batch) } // We will likely have changed our local index, but that won't trigger a diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go index 3754b347..cccd3ebb 100644 --- a/lib/model/folder_recvonly_test.go +++ b/lib/model/folder_recvonly_test.go @@ -28,8 +28,8 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { // Get us a model up and running - m, fcfg := setupROFolder() - ffs := fcfg.Filesystem() + m, f := setupROFolder() + ffs := f.Filesystem() defer os.Remove(m.cfg.ConfigPath()) defer os.Remove(ffs.URI()) defer m.Stop() @@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { // Send and index update for the known stuff m.Index(device1, "ro", knownFiles) - m.updateLocalsFromScanning("ro", knownFiles) + f.updateLocalsFromScanning(knownFiles) size := m.GlobalSize("ro") if size.Files != 1 || size.Directories != 1 { @@ -111,8 +111,8 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { // Get us a model up and running - m, fcfg := setupROFolder() - ffs := fcfg.Filesystem() + m, f := setupROFolder() + ffs := f.Filesystem() defer os.Remove(m.cfg.ConfigPath()) defer os.Remove(ffs.URI()) defer m.Stop() @@ -126,7 +126,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { // Send and index update for the known stuff m.Index(device1, "ro", knownFiles) - m.updateLocalsFromScanning("ro", knownFiles) + f.updateLocalsFromScanning(knownFiles) // Start the folder. This will cause a scan. @@ -204,8 +204,8 @@ func TestRecvOnlyUndoChanges(t *testing.T) { // Get us a model up and running - m, fcfg := setupROFolder() - ffs := fcfg.Filesystem() + m, f := setupROFolder() + ffs := f.Filesystem() defer os.Remove(m.cfg.ConfigPath()) defer os.Remove(ffs.URI()) defer m.Stop() @@ -224,7 +224,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) { // Send and index update for the known stuff m.Index(device1, "ro", knownFiles) - m.updateLocalsFromScanning("ro", knownFiles) + f.updateLocalsFromScanning(knownFiles) // Start the folder. This will cause a scan. @@ -316,7 +316,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi return knownFiles } -func setupROFolder() (*model, config.FolderConfiguration) { +func setupROFolder() (*model, *sendOnlyFolder) { w := createTmpWrapper(defaultCfg) fcfg := testFolderConfigTmp() fcfg.ID = "ro" @@ -324,9 +324,16 @@ func setupROFolder() (*model, config.FolderConfiguration) { w.SetFolder(fcfg) m := newModel(w, myID, "syncthing", "dev", db.OpenMemory(), nil) - m.ServeBackground() m.AddFolder(fcfg) - return m, fcfg + f := &sendOnlyFolder{ + folder: folder{ + fset: m.folderFiles[fcfg.ID], + FolderConfiguration: fcfg, + }, + } + m.ServeBackground() + + return m, f } diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index 4180b25d..23f12961 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -49,7 +49,7 @@ func (f *sendOnlyFolder) pull() bool { f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { - f.model.updateLocalsFromPulling(f.folderID, batch) + f.updateLocalsFromPulling(batch) batch = batch[:0] batchSizeBytes = 0 } @@ -85,25 +85,25 @@ func (f *sendOnlyFolder) pull() bool { }) if len(batch) > 0 { - f.model.updateLocalsFromPulling(f.folderID, batch) + f.updateLocalsFromPulling(batch) } return true } -func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) { +func (f *sendOnlyFolder) Override() { f.setState(FolderScanning) batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) batchSizeBytes := 0 - fs.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { need := fi.(protocol.FileInfo) if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { - updateFn(batch) + f.updateLocalsFromScanning(batch) batch = batch[:0] batchSizeBytes = 0 } - have, ok := fs.Get(protocol.LocalDeviceID, need.Name) + have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name) // Don't override files that are in a bad state (ignored, // unsupported, must rescan, ...). if ok && have.IsInvalid() { @@ -126,7 +126,7 @@ func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileI return true }) if len(batch) > 0 { - updateFn(batch) + f.updateLocalsFromScanning(batch) } f.setState(FolderIdle) } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index c374723c..a2ad531f 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -1604,7 +1604,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) { // All updates to file/folder objects that originated remotely // (across the network) use this call to updateLocals - f.model.updateLocalsFromPulling(f.folderID, files) + f.updateLocalsFromPulling(files) if found { f.ReceivedFile(lastFile.Name, lastFile.IsDeleted()) diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 908a84ad..3e53e252 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -94,11 +94,6 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol fcfg := testFolderConfigTmp() model.AddFolder(fcfg) - // Update index - if files != nil { - model.updateLocalsFromScanning("default", files) - } - f := &sendReceiveFolder{ folder: folder{ stateTracker: newStateTracker("default"), @@ -115,6 +110,11 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol } f.fs = fs.NewMtimeFS(f.Filesystem(), db.NewNamespacedKV(model.db, "mtime")) + // Update index + if files != nil { + f.updateLocalsFromScanning(files) + } + // Folders are never actually started, so no initial scan will be done close(f.initialScanFinished) @@ -362,7 +362,7 @@ func TestWeakHash(t *testing.T) { ModifiedS: info.ModTime().Unix() + 1, } - model.updateLocalsFromScanning("default", []protocol.FileInfo{existingFile}) + fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile}) copyChan := make(chan copyBlocksState) pullChan := make(chan pullBlockState, expectBlocks) @@ -440,7 +440,7 @@ func TestCopierCleanup(t *testing.T) { file.Blocks = []protocol.BlockInfo{blocks[1]} file.Version = file.Version.Update(myID.Short()) // Update index (removing old blocks) - m.updateLocalsFromScanning("default", []protocol.FileInfo{file}) + f.updateLocalsFromScanning([]protocol.FileInfo{file}) if m.finder.Iterate(folders, blocks[0].Hash, iterFn) { t.Error("Unexpected block found") @@ -453,7 +453,7 @@ func TestCopierCleanup(t *testing.T) { file.Blocks = []protocol.BlockInfo{blocks[0]} file.Version = file.Version.Update(myID.Short()) // Update index (removing old blocks) - m.updateLocalsFromScanning("default", []protocol.FileInfo{file}) + f.updateLocalsFromScanning([]protocol.FileInfo{file}) if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) { t.Error("Unexpected block found") @@ -878,7 +878,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) { // create local file file := createFile(t, name, ffs) file.Version = protocol.Vector{}.Update(myID.Short()) - m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file}) + f.updateLocalsFromScanning([]protocol.FileInfo{file}) // Simulate remote creating a dir with the same name file.Type = protocol.FileInfoTypeDirectory @@ -913,7 +913,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) { // create local file file := createFile(t, name, ffs) file.Version = protocol.Vector{}.Update(myID.Short()) - m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file}) + f.updateLocalsFromScanning([]protocol.FileInfo{file}) // Simulate remote creating a symlink with the same name file.Type = protocol.FileInfoTypeSymlink diff --git a/lib/model/model.go b/lib/model/model.go index ddd4e074..cfbdf100 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -54,8 +54,8 @@ const ( type service interface { BringToFront(string) - Override(*db.FileSet, func([]protocol.FileInfo)) - Revert(*db.FileSet, func([]protocol.FileInfo)) + Override() + Revert() DelayScan(d time.Duration) SchedulePull() // something relevant changed, we should try a pull Jobs() ([]string, []string) // In progress, Queued @@ -65,6 +65,7 @@ type service interface { CheckHealth() error Errors() []FileError WatchError() error + ForceRescan(file protocol.FileInfo) error GetStatistics() stats.FolderStatistics getState() (folderState, time.Time, error) @@ -1611,17 +1612,19 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem, // The hashes provided part of the request match what we expect to find according // to what we have in the database, yet the content we've read off the filesystem doesn't // Something is fishy, invalidate the file and rescan it. - cf.SetMustRescan(m.shortID) - - // Update the index and tell others // The file will temporarily become invalid, which is ok as the content is messed up. - m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf}) - - if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil { - l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err) - } else { - l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name) + m.fmut.Lock() + runner, ok := m.folderRunners[folder] + m.fmut.Unlock() + if !ok { + l.Debugf("%v recheckFile: %s: %q / %q: Folder stopped before rescan could be scheduled", m, deviceID, folder, name) + return } + if err := runner.ForceRescan(cf); err != nil { + l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err) + return + } + l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name) } func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) { @@ -1644,16 +1647,6 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo return fs.GetGlobal(file) } -type cFiler struct { - m Model - r string -} - -// Implements scanner.CurrentFiler -func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) { - return cf.m.CurrentFolderFile(cf.r, file) -} - // Connection returns the current connection for device, and a boolean whether a connection was found. func (m *model) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) { m.pmut.RLock() @@ -1988,92 +1981,6 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs return f.Sequence, err } -func (m *model) updateLocalsFromScanning(folder string, fs []protocol.FileInfo) { - m.updateLocals(folder, fs) - - m.fmut.RLock() - folderCfg := m.folderCfgs[folder] - m.fmut.RUnlock() - - m.diskChangeDetected(folderCfg, fs, events.LocalChangeDetected) -} - -func (m *model) updateLocalsFromPulling(folder string, fs []protocol.FileInfo) { - m.updateLocals(folder, fs) - - m.fmut.RLock() - folderCfg := m.folderCfgs[folder] - m.fmut.RUnlock() - - m.diskChangeDetected(folderCfg, fs, events.RemoteChangeDetected) -} - -func (m *model) updateLocals(folder string, fs []protocol.FileInfo) { - m.fmut.RLock() - files := m.folderFiles[folder] - m.fmut.RUnlock() - if files == nil { - // The folder doesn't exist. - return - } - files.Update(protocol.LocalDeviceID, fs) - - filenames := make([]string, len(fs)) - for i, file := range fs { - filenames[i] = file.Name - } - - events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{ - "folder": folder, - "items": len(fs), - "filenames": filenames, - "version": files.Sequence(protocol.LocalDeviceID), - }) -} - -func (m *model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) { - for _, file := range files { - if file.IsInvalid() { - continue - } - - objType := "file" - action := "modified" - - switch { - case file.IsDeleted(): - action = "deleted" - - // If our local vector is version 1 AND it is the only version - // vector so far seen for this file then it is a new file. Else if - // it is > 1 it's not new, and if it is 1 but another shortId - // version vector exists then it is new for us but created elsewhere - // so the file is still not new but modified by us. Only if it is - // truly new do we change this to 'added', else we leave it as - // 'modified'. - case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1: - action = "added" - } - - if file.IsSymlink() { - objType = "symlink" - } else if file.IsDirectory() { - objType = "dir" - } - - // Two different events can be fired here based on what EventType is passed into function - events.Default.Log(typeOfEvent, map[string]string{ - "folder": folderCfg.ID, - "folderID": folderCfg.ID, // incorrect, deprecated, kept for historical compliance - "label": folderCfg.Label, - "action": action, - "type": objType, - "path": filepath.FromSlash(file.Name), - "modifiedBy": file.ModifiedBy.String(), - }) - } -} - func (m *model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) { m.pmut.RLock() nc, ok := m.conn[deviceID] @@ -2276,36 +2183,30 @@ func (m *model) Override(folder string) { // Grab the runner and the file set. m.fmut.RLock() - fs, fsOK := m.folderFiles[folder] - runner, runnerOK := m.folderRunners[folder] + runner, ok := m.folderRunners[folder] m.fmut.RUnlock() - if !fsOK || !runnerOK { + if !ok { return } // Run the override, taking updates as if they came from scanning. - runner.Override(fs, func(files []protocol.FileInfo) { - m.updateLocalsFromScanning(folder, files) - }) + runner.Override() } func (m *model) Revert(folder string) { // Grab the runner and the file set. m.fmut.RLock() - fs, fsOK := m.folderFiles[folder] - runner, runnerOK := m.folderRunners[folder] + runner, ok := m.folderRunners[folder] m.fmut.RUnlock() - if !fsOK || !runnerOK { + if !ok { return } // Run the revert, taking updates as if they came from scanning. - runner.Revert(fs, func(files []protocol.FileInfo) { - m.updateLocalsFromScanning(folder, files) - }) + runner.Revert() } // CurrentSequence returns the change version for the given folder. diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index f44712f3..661135d6 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -448,7 +448,8 @@ func TestIssue4841(t *testing.T) { fc.mut.Unlock() // Setup file from remote that was ignored locally - m.updateLocals(defaultFolderConfig.ID, []protocol.FileInfo{{ + folder := m.folderRunners[defaultFolderConfig.ID].(*sendReceiveFolder) + folder.updateLocals([]protocol.FileInfo{{ Name: "foo", Type: protocol.FileInfoTypeFile, LocalFlags: protocol.FlagLocalIgnored,