perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model: Don't info log repeat pull errors (#6149)

Browse Source
This commit is contained in:
Simon Frei
2019-11-19 09:56:53 +01:00
committed by GitHub
parent 28edf2f5bb
commit 0d14ee4142
10 changed files with 77 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/model/fakeconns_test.go
View File

@@ -33,7 +33,7 @@ type fakeConnection struct {
folder string
model *model
indexFn func(string, []protocol.FileInfo)
requestFn func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
requestFn func(ctx context.Context, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
closeFn func(error)
mut sync.Mutex
}
@@ -82,11 +82,11 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
return nil
}
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
func (f *fakeConnection) Request(ctx context.Context, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
f.mut.Lock()
defer f.mut.Unlock()
if f.requestFn != nil {
return f.requestFn(folder, name, offset, size, hash, fromTemporary)
return f.requestFn(ctx, folder, name, offset, size, hash, fromTemporary)
}
return f.fileData[name], nil
}

6
lib/model/folder.go
View File

@@ -15,6 +15,8 @@ import (
"sync/atomic"
"time"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
@@ -278,7 +280,7 @@ func (f *folder) getHealthError() error {
dbPath := locations.Get(locations.Database)
if usage, err := fs.NewFilesystem(fs.FilesystemTypeBasic, dbPath).Usage("."); err == nil {
if err = config.CheckFreeSpace(f.model.cfg.Options().MinHomeDiskFree, usage); err != nil {
return fmt.Errorf("insufficient space on disk for database (%v): %v", dbPath, err)
return errors.Wrapf(err, "insufficient space on disk for database (%v)", dbPath)
}
}
@@ -297,7 +299,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
oldHash := f.ignores.Hash()
if err := f.ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
err = fmt.Errorf("loading ignores: %v", err)
err = errors.Wrap(err, "loading ignores")
f.setError(err)
return err
}

48
lib/model/folder_sendrecv.go
View File

@@ -104,7 +104,8 @@ type sendReceiveFolder struct {
queue *jobQueue
pullErrors map[string]string // path -> error string
pullErrors map[string]string // errors for most recent/current iteration
oldPullErrors map[string]string // errors from previous iterations for log filtering only
pullErrorsMut sync.Mutex
}
@@ -169,7 +170,7 @@ func (f *sendReceiveFolder) pull() bool {
}
}()
if err := f.ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
err = fmt.Errorf("loading ignores: %v", err)
err = errors.Wrap(err, "loading ignores")
f.setError(err)
return false
}
@@ -210,9 +211,10 @@ func (f *sendReceiveFolder) pull() bool {
}
f.pullErrorsMut.Lock()
hasPullErrs := len(f.pullErrors) > 0
pullErrNum := len(f.pullErrors)
f.pullErrorsMut.Unlock()
if hasPullErrs {
if pullErrNum > 0 {
l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
f.evLogger.Log(events.FolderErrors, map[string]interface{}{
"folder": f.folderID,
"errors": f.Errors(),
@@ -227,6 +229,11 @@ func (f *sendReceiveFolder) pull() bool {
// might have failed). One puller iteration handles all files currently
// flagged as needed in the folder.
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
f.pullErrorsMut.Lock()
f.oldPullErrors = f.pullErrors
f.pullErrors = make(map[string]string)
f.pullErrorsMut.Unlock()
pullChan := make(chan pullBlockState)
copyChan := make(chan copyBlocksState)
finisherChan := make(chan *sharedPullerState)
@@ -269,9 +276,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
doneWg.Done()
}()
// Clear out all previous errors
f.clearPullErrors()
changed, fileDeletions, dirDeletions, err := f.processNeeded(dbUpdateChan, copyChan, scanChan)
// Signal copy and puller routines that we are done with the in data for
@@ -294,6 +298,10 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
close(dbUpdateChan)
updateWg.Wait()
f.pullErrorsMut.Lock()
f.oldPullErrors = nil
f.pullErrorsMut.Unlock()
return changed
}
@@ -1435,7 +1443,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
select {
case <-f.ctx.Done():
state.fail(errors.Wrap(f.ctx.Err(), "folder stopped"))
return
break
default:
}
@@ -1458,7 +1466,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
// leastBusy can select another device when someone else asks.
activity.using(selected)
var buf []byte
buf, lastError = f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
buf, lastError = f.model.requestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
activity.done(selected)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
@@ -1757,6 +1765,11 @@ func (f *sendReceiveFolder) moveForConflict(name, lastModBy string, scanChan cha
}
func (f *sendReceiveFolder) newPullError(path string, err error) {
if errors.Cause(err) == f.ctx.Err() {
// Error because the folder stopped - no point logging/tracking
return
}
f.pullErrorsMut.Lock()
defer f.pullErrorsMut.Unlock()
@@ -1767,18 +1780,19 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
return
}
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
// Establish context to differentiate from errors while scanning.
// Use "syncing" as opposed to "pulling" as the latter might be used
// for errors occurring specificly in the puller routine.
f.pullErrors[path] = fmt.Sprintln("syncing:", err)
}
errStr := fmt.Sprintln("syncing:", err)
f.pullErrors[path] = errStr
func (f *sendReceiveFolder) clearPullErrors() {
f.pullErrorsMut.Lock()
f.pullErrors = make(map[string]string)
f.pullErrorsMut.Unlock()
if oldErr, ok := f.oldPullErrors[path]; ok && oldErr == errStr {
l.Debugf("Repeat error on puller (folder %s, item %q): %v", f.Description(), path, err)
delete(f.oldPullErrors, path) // Potential repeats are now caught by f.pullErrors itself
return
}
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
}
func (f *sendReceiveFolder) Errors() []FileError {

12
lib/model/model.go
View File

@@ -8,6 +8,7 @@ package model
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
@@ -431,18 +432,13 @@ func (m *model) stopFolder(cfg config.FolderConfiguration, err error) {
tokens := m.folderRunnerTokens[cfg.ID]
m.fmut.RUnlock()
// Close connections to affected devices
// Must happen before stopping the folder service to abort ongoing
// transmissions and thus allow timely service termination.
w := m.closeConns(cfg.DeviceIDs(), err)
for _, id := range tokens {
m.RemoveAndWait(id, 0)
}
// Wait for connections to stop to ensure that no more calls to methods
// expecting this folder to exist happen (e.g. .IndexUpdate).
w.Wait()
m.closeConns(cfg.DeviceIDs(), err).Wait()
}
// Need to hold lock on m.fmut when calling this.
@@ -2103,7 +2099,7 @@ func (s *indexSender) sendIndexTo() error {
return err
}
func (m *model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
func (m *model) requestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
m.pmut.RLock()
nc, ok := m.conn[deviceID]
m.pmut.RUnlock()
@@ -2114,7 +2110,7 @@ func (m *model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, offset, size, hash, weakHash, fromTemporary)
return nc.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
return nc.Request(ctx, folder, name, offset, size, hash, weakHash, fromTemporary)
}
func (m *model) ScanFolders() map[string]error {

3
lib/model/model_test.go
View File

@@ -8,6 +8,7 @@ package model
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
@@ -254,7 +255,7 @@ func BenchmarkRequestOut(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
data, err := m.requestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
if err != nil {
b.Error(err)
}

7
lib/model/requests_test.go
View File

@@ -8,6 +8,7 @@ package model
import (
"bytes"
"context"
"errors"
"io/ioutil"
"os"
@@ -136,7 +137,7 @@ func TestSymlinkTraversalWrite(t *testing.T) {
}
}
}
fc.requestFn = func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
fc.requestFn = func(_ context.Context, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
if name != "symlink" && strings.HasPrefix(name, "symlink") {
badReq <- name
}
@@ -411,7 +412,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
}
// Make sure pulling doesn't interfere, as index updates are racy and
// thus we cannot distinguish between scan and pull results.
fc.requestFn = func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
fc.requestFn = func(_ context.Context, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
return nil, nil
}
fc.mut.Unlock()
@@ -996,7 +997,7 @@ func TestNeedFolderFiles(t *testing.T) {
errPreventSync := errors.New("you aren't getting any of this")
fc.mut.Lock()
fc.requestFn = func(string, string, int64, int, []byte, bool) ([]byte, error) {
fc.requestFn = func(context.Context, string, string, int64, int, []byte, bool) ([]byte, error) {
return nil, errPreventSync
}
fc.mut.Unlock()