gui: New rest endpoint to get errors when web UI is opened

Since #4340 pulls aren't happening every 10s anymore and may be delayed up to 1h.
This means that no folder error event reaches the web UI for a long time, thus no
failed items will show up for a long time. Now errors are populated when the
web UI is opened.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4650
LGTM: AudriusButkevicius

cmd/syncthing/gui.go

@@ -111,6 +111,7 @@ type modelIntf interface {
 	RemoteSequence(folder string) (int64, bool)
 	State(folder string) (string, time.Time, error)
 	UsageReportingStats(version int, preview bool) map[string]interface{}
+	PullErrors(folder string) ([]model.FileError, error)
 }
 
 type configIntf interface {
@@ -263,6 +264,7 @@ func (s *apiService) Serve() {
 	getRestMux.HandleFunc("/rest/db/status", s.getDBStatus)                      // folder
 	getRestMux.HandleFunc("/rest/db/browse", s.getDBBrowse)                      // folder [prefix] [dirsonly] [levels]
 	getRestMux.HandleFunc("/rest/folder/versions", s.getFolderVersions)          // folder
+	getRestMux.HandleFunc("/rest/folder/pullerrors", s.getPullErrors)            // folder
 	getRestMux.HandleFunc("/rest/events", s.getIndexEvents)                      // [since] [limit] [timeout] [events]
 	getRestMux.HandleFunc("/rest/events/disk", s.getDiskEvents)                  // [since] [limit] [timeout]
 	getRestMux.HandleFunc("/rest/stats/device", s.getDeviceStats)                // -
@@ -681,12 +683,23 @@ func jsonCompletion(comp model.FolderCompletion) map[string]interface{} {
 func (s *apiService) getDBStatus(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
-	sendJSON(w, folderSummary(s.cfg, s.model, folder))
+	if sum, err := folderSummary(s.cfg, s.model, folder); err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	} else {
+		sendJSON(w, sum)
+	}
 }
 
-func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interface{} {
+func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]interface{}, error) {
 	var res = make(map[string]interface{})
 
+	pullErrors, err := m.PullErrors(folder)
+	if err != nil && err != model.ErrFolderPaused {
+		// Stats from the db can still be obtained if the folder is just paused
+		return nil, err
+	}
+	res["pullErrors"] = len(pullErrors)
+
 	res["invalid"] = "" // Deprecated, retains external API for now
 
 	global := m.GlobalSize(folder)
@@ -700,7 +713,6 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interf
 
 	res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes
 
-	var err error
 	res["state"], res["stateChanged"], err = m.State(folder)
 	if err != nil {
 		res["error"] = err.Error()
@@ -721,7 +733,7 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interf
 		}
 	}
 
-	return res
+	return res, nil
 }
 
 func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) {
@@ -1352,6 +1364,36 @@ func (s *apiService) postFolderVersionsRestore(w http.ResponseWriter, r *http.Re
 	sendJSON(w, ferr)
 }
 
+func (s *apiService) getPullErrors(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	folder := qs.Get("folder")
+	page, perpage := getPagingParams(qs)
+
+	errors, err := s.model.PullErrors(folder)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	start := (page - 1) * perpage
+	if start >= len(errors) {
+		errors = nil
+	} else {
+		errors = errors[start:]
+		if perpage < len(errors) {
+			errors = errors[:perpage]
+		}
+	}
+
+	sendJSON(w, map[string]interface{}{
+		"folder":  folder,
+		"errors":  errors,
+		"page":    page,
+		"perpage": perpage,
+	})
+}
+
 func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	current := qs.Get("current")

cmd/syncthing/mocked_model_test.go

@@ -132,3 +132,7 @@ func (m *mockedModel) State(folder string) (string, time.Time, error) {
 func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
 	return nil
 }
+
+func (m *mockedModel) PullErrors(folder string) ([]model.FileError, error) {
+	return nil, nil
+}

cmd/syncthing/summaryservice.go

@@ -187,7 +187,10 @@ func (c *folderSummaryService) foldersToHandle() []string {
 func (c *folderSummaryService) sendSummary(folder string) {
 	// The folder summary contains how many bytes, files etc
 	// are in the folder and how in sync we are.
-	data := folderSummary(c.cfg, c.model, folder)
+	data, err := folderSummary(c.cfg, c.model, folder)
+	if err != nil {
+		return
+	}
 	events.Default.Log(events.FolderSummary, map[string]interface{}{
 		"folder":  folder,
 		"summary": data,

gui/default/index.html

@@ -355,7 +355,7 @@
                       <th><span class="fa fa-fw fa-exclamation-circle"></span>&nbsp;<span translate>Failed Items</span></th>
                       <!-- Show the number of failed items as a link to bring up the list. -->
                       <td class="text-right">
-                        <a href="" ng-click="showFailed(folder.id)">{{failed[folder.id].length | alwaysNumber}}&nbsp;<span translate>items</span></a>
+                        <a href="" ng-click="showFailed(folder.id)">{{model[folder.id].pullErrors | alwaysNumber}}&nbsp;<span translate>items</span></a>
                       </td>
                     </tr>
                     <tr ng-if="folder.type != 'readwrite'">

gui/default/syncthing/core/syncthingController.js

@@ -48,8 +48,6 @@ angular.module('syncthing.core')
         $scope.neededCurrentPage = 1;
         $scope.neededPageSize = 10;
         $scope.failed = {};
-        $scope.failedCurrentPage = 1;
-        $scope.failedPageSize = 10;
         $scope.scanProgress = {};
         $scope.themes = [];
         $scope.globalChangeEvents = {};
@@ -198,13 +196,6 @@ angular.module('syncthing.core')
                 $scope.model[data.folder].state = data.to;
                 $scope.model[data.folder].error = data.error;
 
-                // If a folder has started syncing, then any old list of
-                // errors is obsolete. We may get a new list of errors very
-                // shortly though.
-                if (data.to === 'syncing') {
-                    $scope.failed[data.folder] = [];
-                }
-
                 // If a folder has started scanning, then any scan progress is
                 // also obsolete.
                 if (data.to === 'scanning') {
@@ -344,8 +335,7 @@ angular.module('syncthing.core')
         });
 
         $scope.$on(Events.FOLDER_ERRORS, function (event, arg) {
-            var data = arg.data;
+            $scope.model[arg.data.folder].pullErrors = arg.data.errors.length;
-            $scope.failed[data.folder] = data.errors;
         });
 
         $scope.$on(Events.FOLDER_SCAN_PROGRESS, function (event, arg) {
@@ -657,12 +647,12 @@ angular.module('syncthing.core')
             refreshNeed($scope.neededFolder);
         };
 
-        $scope.failedPageChanged = function (page) {
+        $scope.refreshFailed = function (page, perpage) {
-            $scope.failedCurrentPage = page;
+            var url = urlbase + '/folder/pullerrors?folder=' + encodeURIComponent($scope.failed.folder);
-        };
+            url += "&page=" + page + "&perpage=" + perpage;
-
+            $http.get(url).success(function (data) {
-        $scope.failedChangePageSize = function (perpage) {
+                $scope.failed = data;
-            $scope.failedPageSize = perpage;
+            }).error($scope.emitHTTPError);
         };
 
         $scope.refreshRemoteNeed = function (folder, page, perpage) {
@@ -1018,14 +1008,6 @@ angular.module('syncthing.core')
             return '?';
         };
 
-        $scope.deviceCompletion = function (deviceCfg) {
-            var conn = $scope.connections[deviceCfg.deviceID];
-            if (conn) {
-                return conn.completion + '%';
-            }
-            return '';
-        };
-
         $scope.friendlyNameFromShort = function (shortID) {
             var matches = $scope.devices.filter(function (n) {
                 return n.deviceID.substr(0, 7) === shortID;
@@ -2067,24 +2049,18 @@ angular.module('syncthing.core')
         };
 
         $scope.showFailed = function (folder) {
-            $scope.failedCurrent = $scope.failed[folder];
+            $scope.failed.folder = folder;
-            $scope.failedFolderPath = $scope.folders[folder].path;
+            $scope.failed = $scope.refreshFailed(1, 10);
-            if ($scope.failedFolderPath[$scope.failedFolderPath.length - 1] !== $scope.system.pathSeparator) {
-                $scope.failedFolderPath += $scope.system.pathSeparator;
-            }
             $('#failed').modal().on('hidden.bs.modal', function () {
-                $scope.failedCurrent = undefined;
+                $scope.failed = {};
             });
         };
 
         $scope.hasFailedFiles = function (folder) {
-            if (!$scope.failed[folder]) {
+            if (!$scope.model[folder]) {
                 return false;
             }
-            if ($scope.failed[folder].length === 0) {
+            return $scope.model[folder].pullErrors !== 0;
-                return false;
-            }
-            return true;
         };
 
         $scope.override = function (folder) {

gui/default/syncthing/transfer/failedFilesModalView.html

@@ -5,15 +5,15 @@
       <span translate>They are retried automatically and will be synced when the error is resolved.</span>
     </p>
     <table class="table table-striped table-dynamic">
-      <tr dir-paginate="e in failedCurrent | itemsPerPage: failedPageSize" current-page="failedCurrentPage" pagination-id="failed">
+      <tr dir-paginate="e in failed.errors | itemsPerPage: failed.perpage" current-page="failed.page" total-items="model[failed.folder].pullErrors" pagination-id="failed">
-        <td>{{failedFolderPath}}{{e.path}}</td>
+        <td>{{e.path}}</td>
         <td><abbr tooltip data-original-title="{{e.error}}">{{e.error | lastErrorComponent}}</abbr></td>
       </tr>
     </table>
-    <dir-pagination-controls on-page-change="failedPageChanged(newPageNumber)" pagination-id="failed"></dir-pagination-controls>
+    <dir-pagination-controls on-page-change="refreshFailed(newPageNumber, failed.perpage)" pagination-id="failed"></dir-pagination-controls>
     <ul class="pagination pull-right">
-      <li ng-repeat="option in [10, 25, 50]" ng-class="{ active: failedPageSize == option }">
+      <li ng-repeat="option in [10, 25, 50]" ng-class="{ active: failed.page == option }">
-        <a href="#" ng-click="failedChangePageSize(option)">{{option}}</a>
+        <a href="#" ng-click="refreshFailed(failed.page, option)">{{option}}</a>
       </li>
     </ul>
     <div class="clearfix"></div>

lib/model/model.go

@@ -64,6 +64,7 @@ type service interface {
 	Serve()
 	Stop()
 	CheckHealth() error
+	PullErrors() []FileError
 
 	getState() (folderState, time.Time, error)
 	setState(state folderState)
@@ -119,7 +120,7 @@ var (
 	errDeviceUnknown     = errors.New("unknown device")
 	errDevicePaused      = errors.New("device is paused")
 	errDeviceIgnored     = errors.New("device is ignored")
-	errFolderPaused      = errors.New("folder is paused")
+	ErrFolderPaused      = errors.New("folder is paused")
 	errFolderNotRunning  = errors.New("folder is not running")
 	errFolderMissing     = errors.New("no such folder")
 	errNetworkNotAllowed = errors.New("network not allowed")
@@ -2226,6 +2227,15 @@ func (m *Model) State(folder string) (string, time.Time, error) {
 	return state.String(), changed, err
 }
 
+func (m *Model) PullErrors(folder string) ([]FileError, error) {
+	m.fmut.RLock()
+	defer m.fmut.RUnlock()
+	if err := m.checkFolderRunningLocked(folder); err != nil {
+		return nil, err
+	}
+	return m.folderRunners[folder].PullErrors(), nil
+}
+
 func (m *Model) Override(folder string) {
 	m.fmut.RLock()
 	fs, ok := m.folderFiles[folder]
@@ -2657,7 +2667,7 @@ func (m *Model) checkFolderRunningLocked(folder string) error {
 	if cfg, ok := m.cfg.Folder(folder); !ok {
 		return errFolderMissing
 	} else if cfg.Paused {
-		return errFolderPaused
+		return ErrFolderPaused
 	}
 
 	return errFolderNotRunning

lib/model/model_test.go

@@ -3229,7 +3229,7 @@ func TestPausedFolders(t *testing.T) {
 	}
 	w.Wait()
 
-	if err := m.ScanFolder("default"); err != errFolderPaused {
+	if err := m.ScanFolder("default"); err != ErrFolderPaused {
 		t.Errorf("Expected folder paused error, received: %v", err)
 	}
 

lib/model/rofolder.go

@@ -66,3 +66,7 @@ func (f *sendOnlyFolder) Serve() {
 func (f *sendOnlyFolder) String() string {
 	return fmt.Sprintf("sendOnlyFolder/%s@%p", f.folderID, f)
 }
+
+func (f *sendOnlyFolder) PullErrors() []FileError {
+	return nil
+}

lib/model/rwfolder.go

@@ -286,7 +286,7 @@ func (f *sendReceiveFolder) pull(prevIgnoreHash string) (curIgnoreHash string, s
 			// we're not making it. Probably there are write
 			// errors preventing us. Flag this with a warning and
 			// wait a bit longer before retrying.
-			if folderErrors := f.currentErrors(); len(folderErrors) > 0 {
+			if folderErrors := f.PullErrors(); len(folderErrors) > 0 {
 				events.Default.Log(events.FolderErrors, map[string]interface{}{
 					"folder": f.folderID,
 					"errors": folderErrors,
@@ -1797,11 +1797,11 @@ func (f *sendReceiveFolder) clearErrors() {
 	f.errorsMut.Unlock()
 }
 
-func (f *sendReceiveFolder) currentErrors() []fileError {
+func (f *sendReceiveFolder) PullErrors() []FileError {
 	f.errorsMut.Lock()
-	errors := make([]fileError, 0, len(f.errors))
+	errors := make([]FileError, 0, len(f.errors))
 	for path, err := range f.errors {
-		errors = append(errors, fileError{path, err})
+		errors = append(errors, FileError{path, err})
 	}
 	sort.Sort(fileErrorList(errors))
 	f.errorsMut.Unlock()
@@ -1880,13 +1880,13 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC
 	return err
 }
 
-// A []fileError is sent as part of an event and will be JSON serialized.
+// A []FileError is sent as part of an event and will be JSON serialized.
-type fileError struct {
+type FileError struct {
 	Path string `json:"path"`
 	Err  string `json:"error"`
 }
 
-type fileErrorList []fileError
+type fileErrorList []FileError
 
 func (l fileErrorList) Len() int {
 	return len(l)