lib/model: Update db on scan/pull in folder (#5608)

lib/model/folder.go

@@ -198,9 +198,9 @@ func (f *folder) Serve() {
 
 func (f *folder) BringToFront(string) {}
 
-func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
+func (f *folder) Override() {}
 
-func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
+func (f *folder) Revert() {}
 
 func (f *folder) DelayScan(next time.Duration) {
 	f.Delay(next)
@@ -345,7 +345,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		Subs:                  subDirs,
 		Matcher:               f.ignores,
 		TempLifetime:          time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
-		CurrentFiler:          cFiler{f.model, f.ID},
+		CurrentFiler:          cFiler{f.fset},
 		Filesystem:            mtimefs,
 		IgnorePerms:           f.IgnorePerms,
 		AutoNormalize:         f.AutoNormalize,
@@ -361,7 +361,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 			l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
 			return err
 		}
-		f.model.updateLocalsFromScanning(f.ID, fs)
+		f.updateLocalsFromScanning(fs)
 		return nil
 	}
 	// Resolve items which are identical with the global state.
@@ -737,6 +737,86 @@ func (f *folder) Errors() []FileError {
 	return append([]FileError{}, f.scanErrors...)
 }
 
+// ForceRescan marks the file such that it gets rehashed on next scan and then
+// immediately executes that scan.
+func (f *folder) ForceRescan(file protocol.FileInfo) error {
+	file.SetMustRescan(f.shortID)
+	f.fset.Update(protocol.LocalDeviceID, []protocol.FileInfo{file})
+
+	return f.Scan([]string{file.Name})
+}
+
+func (f *folder) updateLocalsFromScanning(fs []protocol.FileInfo) {
+	f.updateLocals(fs)
+
+	f.emitDiskChangeEvents(fs, events.LocalChangeDetected)
+}
+
+func (f *folder) updateLocalsFromPulling(fs []protocol.FileInfo) {
+	f.updateLocals(fs)
+
+	f.emitDiskChangeEvents(fs, events.RemoteChangeDetected)
+}
+
+func (f *folder) updateLocals(fs []protocol.FileInfo) {
+	f.fset.Update(protocol.LocalDeviceID, fs)
+
+	filenames := make([]string, len(fs))
+	for i, file := range fs {
+		filenames[i] = file.Name
+	}
+
+	events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
+		"folder":    f.ID,
+		"items":     len(fs),
+		"filenames": filenames,
+		"version":   f.fset.Sequence(protocol.LocalDeviceID),
+	})
+}
+
+func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events.EventType) {
+	for _, file := range fs {
+		if file.IsInvalid() {
+			continue
+		}
+
+		objType := "file"
+		action := "modified"
+
+		switch {
+		case file.IsDeleted():
+			action = "deleted"
+
+		// If our local vector is version 1 AND it is the only version
+		// vector so far seen for this file then it is a new file.  Else if
+		// it is > 1 it's not new, and if it is 1 but another shortId
+		// version vector exists then it is new for us but created elsewhere
+		// so the file is still not new but modified by us. Only if it is
+		// truly new do we change this to 'added', else we leave it as
+		// 'modified'.
+		case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1:
+			action = "added"
+		}
+
+		if file.IsSymlink() {
+			objType = "symlink"
+		} else if file.IsDirectory() {
+			objType = "dir"
+		}
+
+		// Two different events can be fired here based on what EventType is passed into function
+		events.Default.Log(typeOfEvent, map[string]string{
+			"folder":     f.ID,
+			"folderID":   f.ID, // incorrect, deprecated, kept for historical compliance
+			"label":      f.Label,
+			"action":     action,
+			"type":       objType,
+			"path":       filepath.FromSlash(file.Name),
+			"modifiedBy": file.ModifiedBy.String(),
+		})
+	}
+}
+
 // The exists function is expected to return true for all known paths
 // (excluding "" and ".")
 func unifySubs(dirs []string, exists func(dir string) bool) []string {
@@ -770,3 +850,12 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
 	}
 	return dirs
 }
+
+type cFiler struct {
+	*db.FileSet
+}
+
+// Implements scanner.CurrentFiler
+func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
+	return cf.Get(protocol.LocalDeviceID, file)
+}

lib/model/folder_recvonly.go

@@ -62,7 +62,7 @@ func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matche
 	return &receiveOnlyFolder{sr}
 }
 
-func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
+func (f *receiveOnlyFolder) Revert() {
 	f.setState(FolderScanning)
 	defer f.setState(FolderIdle)
 
@@ -78,7 +78,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
 
 	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
 	batchSizeBytes := 0
-	fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		fi := intf.(protocol.FileInfo)
 		if !fi.IsReceiveOnlyChanged() {
 			// We're only interested in files that have changed locally in
@@ -124,14 +124,14 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
 		batchSizeBytes += fi.ProtoSize()
 
 		if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes {
-			updateFn(batch)
+			f.updateLocalsFromScanning(batch)
 			batch = batch[:0]
 			batchSizeBytes = 0
 		}
 		return true
 	})
 	if len(batch) > 0 {
-		updateFn(batch)
+		f.updateLocalsFromScanning(batch)
 	}
 	batch = batch[:0]
 	batchSizeBytes = 0
@@ -153,7 +153,7 @@ func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.File
 		})
 	}
 	if len(batch) > 0 {
-		updateFn(batch)
+		f.updateLocalsFromScanning(batch)
 	}
 
 	// We will likely have changed our local index, but that won't trigger a

lib/model/folder_recvonly_test.go

@@ -28,8 +28,8 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 
 	// Get us a model up and running
 
-	m, fcfg := setupROFolder()
+	m, f := setupROFolder()
-	ffs := fcfg.Filesystem()
+	ffs := f.Filesystem()
 	defer os.Remove(m.cfg.ConfigPath())
 	defer os.Remove(ffs.URI())
 	defer m.Stop()
@@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 	// Send and index update for the known stuff
 
 	m.Index(device1, "ro", knownFiles)
-	m.updateLocalsFromScanning("ro", knownFiles)
+	f.updateLocalsFromScanning(knownFiles)
 
 	size := m.GlobalSize("ro")
 	if size.Files != 1 || size.Directories != 1 {
@@ -111,8 +111,8 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 
 	// Get us a model up and running
 
-	m, fcfg := setupROFolder()
+	m, f := setupROFolder()
-	ffs := fcfg.Filesystem()
+	ffs := f.Filesystem()
 	defer os.Remove(m.cfg.ConfigPath())
 	defer os.Remove(ffs.URI())
 	defer m.Stop()
@@ -126,7 +126,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 	// Send and index update for the known stuff
 
 	m.Index(device1, "ro", knownFiles)
-	m.updateLocalsFromScanning("ro", knownFiles)
+	f.updateLocalsFromScanning(knownFiles)
 
 	// Start the folder. This will cause a scan.
 
@@ -204,8 +204,8 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	// Get us a model up and running
 
-	m, fcfg := setupROFolder()
+	m, f := setupROFolder()
-	ffs := fcfg.Filesystem()
+	ffs := f.Filesystem()
 	defer os.Remove(m.cfg.ConfigPath())
 	defer os.Remove(ffs.URI())
 	defer m.Stop()
@@ -224,7 +224,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 	// Send and index update for the known stuff
 
 	m.Index(device1, "ro", knownFiles)
-	m.updateLocalsFromScanning("ro", knownFiles)
+	f.updateLocalsFromScanning(knownFiles)
 
 	// Start the folder. This will cause a scan.
 
@@ -316,7 +316,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
 	return knownFiles
 }
 
-func setupROFolder() (*model, config.FolderConfiguration) {
+func setupROFolder() (*model, *sendOnlyFolder) {
 	w := createTmpWrapper(defaultCfg)
 	fcfg := testFolderConfigTmp()
 	fcfg.ID = "ro"
@@ -324,9 +324,16 @@ func setupROFolder() (*model, config.FolderConfiguration) {
 	w.SetFolder(fcfg)
 
 	m := newModel(w, myID, "syncthing", "dev", db.OpenMemory(), nil)
-	m.ServeBackground()
 	m.AddFolder(fcfg)
 
-	return m, fcfg
+	f := &sendOnlyFolder{
-
+		folder: folder{
+			fset:                m.folderFiles[fcfg.ID],
+			FolderConfiguration: fcfg,
+		},
+	}
+
+	m.ServeBackground()
+
+	return m, f
 }

lib/model/folder_sendonly.go

@@ -49,7 +49,7 @@ func (f *sendOnlyFolder) pull() bool {
 
 	f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
-			f.model.updateLocalsFromPulling(f.folderID, batch)
+			f.updateLocalsFromPulling(batch)
 			batch = batch[:0]
 			batchSizeBytes = 0
 		}
@@ -85,25 +85,25 @@ func (f *sendOnlyFolder) pull() bool {
 	})
 
 	if len(batch) > 0 {
-		f.model.updateLocalsFromPulling(f.folderID, batch)
+		f.updateLocalsFromPulling(batch)
 	}
 
 	return true
 }
 
-func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
+func (f *sendOnlyFolder) Override() {
 	f.setState(FolderScanning)
 	batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
 	batchSizeBytes := 0
-	fs.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+	f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
-			updateFn(batch)
+			f.updateLocalsFromScanning(batch)
 			batch = batch[:0]
 			batchSizeBytes = 0
 		}
 
-		have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
+		have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name)
 		// Don't override files that are in a bad state (ignored,
 		// unsupported, must rescan, ...).
 		if ok && have.IsInvalid() {
@@ -126,7 +126,7 @@ func (f *sendOnlyFolder) Override(fs *db.FileSet, updateFn func([]protocol.FileI
 		return true
 	})
 	if len(batch) > 0 {
-		updateFn(batch)
+		f.updateLocalsFromScanning(batch)
 	}
 	f.setState(FolderIdle)
 }

lib/model/folder_sendrecv.go

@@ -1604,7 +1604,7 @@ func (f *sendReceiveFolder) dbUpdaterRoutine(dbUpdateChan <-chan dbUpdateJob) {
 
 		// All updates to file/folder objects that originated remotely
 		// (across the network) use this call to updateLocals
-		f.model.updateLocalsFromPulling(f.folderID, files)
+		f.updateLocalsFromPulling(files)
 
 		if found {
 			f.ReceivedFile(lastFile.Name, lastFile.IsDeleted())

lib/model/folder_sendrecv_test.go

@@ -94,11 +94,6 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
 	fcfg := testFolderConfigTmp()
 	model.AddFolder(fcfg)
 
-	// Update index
-	if files != nil {
-		model.updateLocalsFromScanning("default", files)
-	}
-
 	f := &sendReceiveFolder{
 		folder: folder{
 			stateTracker:        newStateTracker("default"),
@@ -115,6 +110,11 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
 	}
 	f.fs = fs.NewMtimeFS(f.Filesystem(), db.NewNamespacedKV(model.db, "mtime"))
 
+	// Update index
+	if files != nil {
+		f.updateLocalsFromScanning(files)
+	}
+
 	// Folders are never actually started, so no initial scan will be done
 	close(f.initialScanFinished)
 
@@ -362,7 +362,7 @@ func TestWeakHash(t *testing.T) {
 		ModifiedS: info.ModTime().Unix() + 1,
 	}
 
-	model.updateLocalsFromScanning("default", []protocol.FileInfo{existingFile})
+	fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile})
 
 	copyChan := make(chan copyBlocksState)
 	pullChan := make(chan pullBlockState, expectBlocks)
@@ -440,7 +440,7 @@ func TestCopierCleanup(t *testing.T) {
 	file.Blocks = []protocol.BlockInfo{blocks[1]}
 	file.Version = file.Version.Update(myID.Short())
 	// Update index (removing old blocks)
-	m.updateLocalsFromScanning("default", []protocol.FileInfo{file})
+	f.updateLocalsFromScanning([]protocol.FileInfo{file})
 
 	if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
@@ -453,7 +453,7 @@ func TestCopierCleanup(t *testing.T) {
 	file.Blocks = []protocol.BlockInfo{blocks[0]}
 	file.Version = file.Version.Update(myID.Short())
 	// Update index (removing old blocks)
-	m.updateLocalsFromScanning("default", []protocol.FileInfo{file})
+	f.updateLocalsFromScanning([]protocol.FileInfo{file})
 
 	if !m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
 		t.Error("Unexpected block found")
@@ -878,7 +878,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
 	// create local file
 	file := createFile(t, name, ffs)
 	file.Version = protocol.Vector{}.Update(myID.Short())
-	m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file})
+	f.updateLocalsFromScanning([]protocol.FileInfo{file})
 
 	// Simulate remote creating a dir with the same name
 	file.Type = protocol.FileInfoTypeDirectory
@@ -913,7 +913,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
 	// create local file
 	file := createFile(t, name, ffs)
 	file.Version = protocol.Vector{}.Update(myID.Short())
-	m.updateLocalsFromScanning(f.ID, []protocol.FileInfo{file})
+	f.updateLocalsFromScanning([]protocol.FileInfo{file})
 
 	// Simulate remote creating a symlink with the same name
 	file.Type = protocol.FileInfoTypeSymlink

lib/model/model.go

@@ -54,8 +54,8 @@ const (
 
 type service interface {
 	BringToFront(string)
-	Override(*db.FileSet, func([]protocol.FileInfo))
+	Override()
-	Revert(*db.FileSet, func([]protocol.FileInfo))
+	Revert()
 	DelayScan(d time.Duration)
 	SchedulePull()              // something relevant changed, we should try a pull
 	Jobs() ([]string, []string) // In progress, Queued
@@ -65,6 +65,7 @@ type service interface {
 	CheckHealth() error
 	Errors() []FileError
 	WatchError() error
+	ForceRescan(file protocol.FileInfo) error
 	GetStatistics() stats.FolderStatistics
 
 	getState() (folderState, time.Time, error)
@@ -1611,17 +1612,19 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem,
 	// The hashes provided part of the request match what we expect to find according
 	// to what we have in the database, yet the content we've read off the filesystem doesn't
 	// Something is fishy, invalidate the file and rescan it.
-	cf.SetMustRescan(m.shortID)
-
-	// Update the index and tell others
 	// The file will temporarily become invalid, which is ok as the content is messed up.
-	m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
+	m.fmut.Lock()
-
+	runner, ok := m.folderRunners[folder]
-	if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
+	m.fmut.Unlock()
-		l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
+	if !ok {
-	} else {
+		l.Debugf("%v recheckFile: %s: %q / %q: Folder stopped before rescan could be scheduled", m, deviceID, folder, name)
-		l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
+		return
 	}
+	if err := runner.ForceRescan(cf); err != nil {
+		l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
+		return
+	}
+	l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
 }
 
 func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
@@ -1644,16 +1647,6 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
 	return fs.GetGlobal(file)
 }
 
-type cFiler struct {
-	m Model
-	r string
-}
-
-// Implements scanner.CurrentFiler
-func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
-	return cf.m.CurrentFolderFile(cf.r, file)
-}
-
 // Connection returns the current connection for device, and a boolean whether a connection was found.
 func (m *model) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) {
 	m.pmut.RLock()
@@ -1988,92 +1981,6 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
 	return f.Sequence, err
 }
 
-func (m *model) updateLocalsFromScanning(folder string, fs []protocol.FileInfo) {
-	m.updateLocals(folder, fs)
-
-	m.fmut.RLock()
-	folderCfg := m.folderCfgs[folder]
-	m.fmut.RUnlock()
-
-	m.diskChangeDetected(folderCfg, fs, events.LocalChangeDetected)
-}
-
-func (m *model) updateLocalsFromPulling(folder string, fs []protocol.FileInfo) {
-	m.updateLocals(folder, fs)
-
-	m.fmut.RLock()
-	folderCfg := m.folderCfgs[folder]
-	m.fmut.RUnlock()
-
-	m.diskChangeDetected(folderCfg, fs, events.RemoteChangeDetected)
-}
-
-func (m *model) updateLocals(folder string, fs []protocol.FileInfo) {
-	m.fmut.RLock()
-	files := m.folderFiles[folder]
-	m.fmut.RUnlock()
-	if files == nil {
-		// The folder doesn't exist.
-		return
-	}
-	files.Update(protocol.LocalDeviceID, fs)
-
-	filenames := make([]string, len(fs))
-	for i, file := range fs {
-		filenames[i] = file.Name
-	}
-
-	events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
-		"folder":    folder,
-		"items":     len(fs),
-		"filenames": filenames,
-		"version":   files.Sequence(protocol.LocalDeviceID),
-	})
-}
-
-func (m *model) diskChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo, typeOfEvent events.EventType) {
-	for _, file := range files {
-		if file.IsInvalid() {
-			continue
-		}
-
-		objType := "file"
-		action := "modified"
-
-		switch {
-		case file.IsDeleted():
-			action = "deleted"
-
-		// If our local vector is version 1 AND it is the only version
-		// vector so far seen for this file then it is a new file.  Else if
-		// it is > 1 it's not new, and if it is 1 but another shortId
-		// version vector exists then it is new for us but created elsewhere
-		// so the file is still not new but modified by us. Only if it is
-		// truly new do we change this to 'added', else we leave it as
-		// 'modified'.
-		case len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1:
-			action = "added"
-		}
-
-		if file.IsSymlink() {
-			objType = "symlink"
-		} else if file.IsDirectory() {
-			objType = "dir"
-		}
-
-		// Two different events can be fired here based on what EventType is passed into function
-		events.Default.Log(typeOfEvent, map[string]string{
-			"folder":     folderCfg.ID,
-			"folderID":   folderCfg.ID, // incorrect, deprecated, kept for historical compliance
-			"label":      folderCfg.Label,
-			"action":     action,
-			"type":       objType,
-			"path":       filepath.FromSlash(file.Name),
-			"modifiedBy": file.ModifiedBy.String(),
-		})
-	}
-}
-
 func (m *model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
 	m.pmut.RLock()
 	nc, ok := m.conn[deviceID]
@@ -2276,36 +2183,30 @@ func (m *model) Override(folder string) {
 	// Grab the runner and the file set.
 
 	m.fmut.RLock()
-	fs, fsOK := m.folderFiles[folder]
+	runner, ok := m.folderRunners[folder]
-	runner, runnerOK := m.folderRunners[folder]
 	m.fmut.RUnlock()
-	if !fsOK || !runnerOK {
+	if !ok {
 		return
 	}
 
 	// Run the override, taking updates as if they came from scanning.
 
-	runner.Override(fs, func(files []protocol.FileInfo) {
+	runner.Override()
-		m.updateLocalsFromScanning(folder, files)
-	})
 }
 
 func (m *model) Revert(folder string) {
 	// Grab the runner and the file set.
 
 	m.fmut.RLock()
-	fs, fsOK := m.folderFiles[folder]
+	runner, ok := m.folderRunners[folder]
-	runner, runnerOK := m.folderRunners[folder]
 	m.fmut.RUnlock()
-	if !fsOK || !runnerOK {
+	if !ok {
 		return
 	}
 
 	// Run the revert, taking updates as if they came from scanning.
 
-	runner.Revert(fs, func(files []protocol.FileInfo) {
+	runner.Revert()
-		m.updateLocalsFromScanning(folder, files)
-	})
 }
 
 // CurrentSequence returns the change version for the given folder.

lib/model/requests_test.go

@@ -448,7 +448,8 @@ func TestIssue4841(t *testing.T) {
 	fc.mut.Unlock()
 
 	// Setup file from remote that was ignored locally
-	m.updateLocals(defaultFolderConfig.ID, []protocol.FileInfo{{
+	folder := m.folderRunners[defaultFolderConfig.ID].(*sendReceiveFolder)
+	folder.updateLocals([]protocol.FileInfo{{
 		Name:       "foo",
 		Type:       protocol.FileInfoTypeFile,
 		LocalFlags: protocol.FlagLocalIgnored,