all: Display fs watcher status and retry starting it (ref #4552) (#4713)

lib/model/folder.go

@@ -8,12 +8,17 @@ package model
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/watchaggregator"
 )
 
+var errWatchNotStarted error = errors.New("not started")
+
 type folder struct {
 	stateTracker
 	config.FolderConfiguration
@@ -26,6 +31,8 @@ type folder struct {
 	watchCancel         context.CancelFunc
 	watchChan           chan []string
 	restartWatchChan    chan struct{}
+	watchErr            error
+	watchErrMut         sync.Mutex
 }
 
 func newFolder(model *Model, cfg config.FolderConfiguration) folder {
@@ -41,6 +48,8 @@ func newFolder(model *Model, cfg config.FolderConfiguration) folder {
 		model:               model,
 		initialScanFinished: make(chan struct{}),
 		watchCancel:         func() {},
+		watchErr:            errWatchNotStarted,
+		watchErrMut:         sync.NewMutex(),
 	}
 }
 
@@ -127,28 +136,22 @@ func (f *folder) scanTimerFired() {
 	f.scan.Reschedule()
 }
 
-func (f *folder) startWatch() {
-	ctx, cancel := context.WithCancel(f.ctx)
-	f.model.fmut.RLock()
-	ignores := f.model.folderIgnores[f.folderID]
-	f.model.fmut.RUnlock()
-	eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
-	if err != nil {
-		l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
-	} else {
-		f.watchChan = make(chan []string)
-		f.watchCancel = cancel
-		watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
-		l.Infoln("Started filesystem watcher for folder", f.Description())
-	}
+func (f *folder) WatchError() error {
+	f.watchErrMut.Lock()
+	defer f.watchErrMut.Unlock()
+	return f.watchErr
 }
 
-func (f *folder) restartWatch() {
+// stopWatch immediately aborts watching and may be called asynchronously
+func (f *folder) stopWatch() {
 	f.watchCancel()
-	f.startWatch()
-	f.Scan(nil)
+	f.watchErrMut.Lock()
+	f.watchErr = errWatchNotStarted
+	f.watchErrMut.Unlock()
 }
 
+// scheduleWatchRestart makes sure watching is restarted from the main for loop
+// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
 func (f *folder) scheduleWatchRestart() {
 	select {
 	case f.restartWatchChan <- struct{}{}:
@@ -159,6 +162,56 @@ func (f *folder) scheduleWatchRestart() {
 	}
 }
 
+// restartWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) restartWatch() {
+	f.stopWatch()
+	f.startWatch()
+	f.Scan(nil)
+}
+
+// startWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) startWatch() {
+	ctx, cancel := context.WithCancel(f.ctx)
+	f.model.fmut.RLock()
+	ignores := f.model.folderIgnores[f.folderID]
+	f.model.fmut.RUnlock()
+	f.watchChan = make(chan []string)
+	f.watchCancel = cancel
+	go f.startWatchAsync(ctx, ignores)
+}
+
+// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
+// It is a convenience function that should not be used except in startWatch.
+func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-timer.C:
+			eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
+			f.watchErrMut.Lock()
+			prevErr := f.watchErr
+			f.watchErr = err
+			f.watchErrMut.Unlock()
+			if err != nil {
+				if prevErr == errWatchNotStarted {
+					l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
+				} else {
+					l.Debugf("Failed to start filesystem watcher for folder %s again: %v", f.Description(), err)
+				}
+				timer.Reset(time.Minute)
+				continue
+			}
+			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
+			l.Debugln("Started filesystem watcher for folder", f.Description())
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (f *folder) setError(err error) {
 	_, _, oldErr := f.getState()
 	if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
@@ -177,7 +230,7 @@ func (f *folder) setError(err error) {
 
 	if f.FSWatcherEnabled {
 		if err != nil {
-			f.watchCancel()
+			f.stopWatch()
 		} else {
 			f.scheduleWatchRestart()
 		}