More graceful handling on folder errors (fixes #762)

Checks health before accepting every scanner batch, also
recovers from errors without having to restart.

internal/model/model.go

@@ -1104,7 +1104,11 @@ func (m *Model) ScanFolders() map[string]error {
 				errorsMut.Lock()
 				errors[folder] = err
 				errorsMut.Unlock()
-				m.cfg.InvalidateFolder(folder, err.Error())
+				// Potentially sets the error twice, once in the scanner just
+				// by doing a check, and once here, if the error returned is
+				// the same one as returned by CheckFolderHealth, though
+				// duplicate set is handled by SetFolderError
+				m.cfg.SetFolderError(folder, err)
 			}
 			wg.Done()
 		}()
@@ -1180,9 +1184,11 @@ nextSub:
 	}
 
 	runner.setState(FolderScanning)
+	defer runner.setState(FolderIdle)
 	fchan, err := w.Walk()
 
 	if err != nil {
+		m.cfg.SetFolderError(folder, err)
 		return err
 	}
 	batchSize := 100
@@ -1196,12 +1202,20 @@ nextSub:
 			"size":     f.Size(),
 		})
 		if len(batch) == batchSize {
+			if err := m.CheckFolderHealth(folder); err != nil {
+				l.Infoln("Stopping folder %s mid-scan due to folder error: %s", folder, err)
+				return err
+			}
 			fs.Update(protocol.LocalDeviceID, batch)
 			batch = batch[:0]
 		}
 		batch = append(batch, f)
 	}
-	if len(batch) > 0 {
+
+	if err := m.CheckFolderHealth(folder); err != nil {
+		l.Infoln("Stopping folder %s mid-scan due to folder error: %s", folder, err)
+		return err
+	} else if len(batch) > 0 {
 		fs.Update(protocol.LocalDeviceID, batch)
 	}
 
@@ -1286,7 +1300,6 @@ nextSub:
 		fs.Update(protocol.LocalDeviceID, batch)
 	}
 
-	runner.setState(FolderIdle)
 	return nil
 }
 
@@ -1510,6 +1523,73 @@ func (m *Model) BringToFront(folder, file string) {
 	}
 }
 
+// Returns current folder error, or nil if the folder is healthy.
+// Updates the Invalid field on the folder configuration struct, and emits a
+// ConfigSaved event which causes a GUI refresh.
+func (m *Model) CheckFolderHealth(id string) error {
+	folder, ok := m.cfg.Folders()[id]
+	if !ok {
+		return errors.New("Folder does not exist")
+	}
+
+	fi, err := os.Stat(folder.Path)
+	if m.CurrentLocalVersion(id) > 0 {
+		// Safety check. If the cached index contains files but the
+		// folder doesn't exist, we have a problem. We would assume
+		// that all files have been deleted which might not be the case,
+		// so mark it as invalid instead.
+		if err != nil || !fi.IsDir() {
+			err = errors.New("Folder path missing")
+		} else if !folder.HasMarker() {
+			err = errors.New("Folder marker missing")
+		}
+	} else if os.IsNotExist(err) {
+		// If we don't have any files in the index, and the directory
+		// doesn't exist, try creating it.
+		err = os.MkdirAll(folder.Path, 0700)
+		if err == nil {
+			err = folder.CreateMarker()
+		}
+	} else if !folder.HasMarker() {
+		// If we don't have any files in the index, and the path does exist
+		// but the marker is not there, create it.
+		err = folder.CreateMarker()
+	}
+
+	if err == nil {
+		if folder.Invalid != "" {
+			l.Infof("Starting folder %q after error %q", folder.ID, folder.Invalid)
+			m.cfg.SetFolderError(id, nil)
+		}
+
+		if folder, ok := m.cfg.Folders()[id]; !ok || folder.Invalid != "" {
+			panic("Unable to unset folder \"" + id + "\" error.")
+		}
+
+		return nil
+	}
+
+	if folder.Invalid == err.Error() {
+		return err
+	}
+
+	// folder is a copy of the original struct, hence Invalid value is
+	// preserved after the set.
+	m.cfg.SetFolderError(id, err)
+
+	if folder.Invalid == "" {
+		l.Warnf("Stopping folder %q - %v", folder.ID, err)
+	} else {
+		l.Infof("Folder %q error changed: %q -> %q", folder.ID, folder.Invalid, err)
+	}
+
+	if folder, ok := m.cfg.Folders()[id]; !ok || folder.Invalid != err.Error() {
+		panic("Unable to set folder \"" + id + "\" error.")
+	}
+
+	return err
+}
+
 func (m *Model) String() string {
 	return fmt.Sprintf("model@%p", m)
 }