From af0bc95de5b992b2c483e052eb822b4f5e97cc61 Mon Sep 17 00:00:00 2001 From: "Lars K.W. Gohlke" Date: Wed, 29 Jun 2016 06:37:34 +0000 Subject: [PATCH] lib/model: Refactor encapsulation of the folder scanning GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3017 --- cmd/syncthing/gui.go | 4 +- cmd/syncthing/mocked_model_test.go | 2 +- lib/model/folder.go | 2 +- lib/model/folderscan.go | 49 ----------------------- lib/model/folderscanner.go | 63 ++++++++++++++++++++++++++++++ lib/model/folderstate.go | 7 ++++ lib/model/model.go | 34 ++++++++-------- lib/model/model_test.go | 2 +- lib/model/rofolder.go | 28 +++++-------- lib/model/rwfolder.go | 18 +++------ lib/model/rwfolder_test.go | 6 +-- 11 files changed, 108 insertions(+), 107 deletions(-) delete mode 100644 lib/model/folderscan.go create mode 100644 lib/model/folderscanner.go diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 8c8d00d8..d6d6d4c3 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -86,7 +86,7 @@ type modelIntf interface { DelayScan(folder string, next time.Duration) ScanFolder(folder string) error ScanFolders() map[string]error - ScanFolderSubs(folder string, subs []string) error + ScanFolderSubdirs(folder string, subs []string) error BringToFront(folder, file string) ConnectedTo(deviceID protocol.DeviceID) bool GlobalSize(folder string) (nfiles, deleted int, bytes int64) @@ -1071,7 +1071,7 @@ func (s *apiService) postDBScan(w http.ResponseWriter, r *http.Request) { } subs := qs["sub"] - err = s.model.ScanFolderSubs(folder, subs) + err = s.model.ScanFolderSubdirs(folder, subs) if err != nil { http.Error(w, err.Error(), 500) return diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 4db67082..9a546ff6 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -85,7 +85,7 @@ func (m *mockedModel) ScanFolders() map[string]error { return nil } -func (m *mockedModel) ScanFolderSubs(folder string, subs []string) error { +func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error { return nil } diff --git a/lib/model/folder.go b/lib/model/folder.go index 8f15adc8..dfe23723 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -10,7 +10,7 @@ import "time" type folder struct { stateTracker - scan folderscan + scan folderScanner model *Model stop chan struct{} } diff --git a/lib/model/folderscan.go b/lib/model/folderscan.go deleted file mode 100644 index 65ed31aa..00000000 --- a/lib/model/folderscan.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -package model - -import ( - "math/rand" - "time" -) - -type rescanRequest struct { - subdirs []string - err chan error -} - -// bundle all folder scan activity -type folderscan struct { - interval time.Duration - timer *time.Timer - now chan rescanRequest - delay chan time.Duration -} - -func (s *folderscan) reschedule() { - if s.interval == 0 { - return - } - // Sleep a random time between 3/4 and 5/4 of the configured interval. - sleepNanos := (s.interval.Nanoseconds()*3 + rand.Int63n(2*s.interval.Nanoseconds())) / 4 - interval := time.Duration(sleepNanos) * time.Nanosecond - l.Debugln(s, "next rescan in", interval) - s.timer.Reset(interval) -} - -func (s *folderscan) Scan(subdirs []string) error { - req := rescanRequest{ - subdirs: subdirs, - err: make(chan error), - } - s.now <- req - return <-req.err -} - -func (s *folderscan) Delay(next time.Duration) { - s.delay <- next -} diff --git a/lib/model/folderscanner.go b/lib/model/folderscanner.go new file mode 100644 index 00000000..d374d1c2 --- /dev/null +++ b/lib/model/folderscanner.go @@ -0,0 +1,63 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package model + +import ( + "github.com/syncthing/syncthing/lib/config" + "math/rand" + "time" +) + +type rescanRequest struct { + subdirs []string + err chan error +} + +// bundle all folder scan activity +type folderScanner struct { + interval time.Duration + timer *time.Timer + now chan rescanRequest + delay chan time.Duration +} + +func newFolderScanner(config config.FolderConfiguration) folderScanner { + return folderScanner{ + interval: time.Duration(config.RescanIntervalS) * time.Second, + timer: time.NewTimer(time.Millisecond), // The first scan should be done immediately. + now: make(chan rescanRequest), + delay: make(chan time.Duration), + } +} + +func (f *folderScanner) Reschedule() { + if f.interval == 0 { + return + } + // Sleep a random time between 3/4 and 5/4 of the configured interval. + sleepNanos := (f.interval.Nanoseconds()*3 + rand.Int63n(2*f.interval.Nanoseconds())) / 4 + interval := time.Duration(sleepNanos) * time.Nanosecond + l.Debugln(f, "next rescan in", interval) + f.timer.Reset(interval) +} + +func (f *folderScanner) Scan(subdirs []string) error { + req := rescanRequest{ + subdirs: subdirs, + err: make(chan error), + } + f.now <- req + return <-req.err +} + +func (f *folderScanner) Delay(next time.Duration) { + f.delay <- next +} + +func (f *folderScanner) HasNoInterval() bool { + return f.interval == 0 +} diff --git a/lib/model/folderstate.go b/lib/model/folderstate.go index f68bee35..19984c0a 100644 --- a/lib/model/folderstate.go +++ b/lib/model/folderstate.go @@ -46,6 +46,13 @@ type stateTracker struct { changed time.Time } +func newStateTracker(id string) stateTracker { + return stateTracker{ + folderID: id, + mut: sync.NewMutex(), + } +} + // setState sets the new folder state, for states other than FolderError. func (s *stateTracker) setState(newState folderState) { if newState == FolderError { diff --git a/lib/model/model.go b/lib/model/model.go index 5bcec120..973c4845 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -47,18 +47,18 @@ const ( ) type service interface { - Serve() - Stop() - Jobs() ([]string, []string) // In progress, Queued BringToFront(string) DelayScan(d time.Duration) - IndexUpdated() // Remote index was updated notification + IndexUpdated() // Remote index was updated notification + Jobs() ([]string, []string) // In progress, Queued Scan(subs []string) error + Serve() + Stop() - setState(state folderState) - setError(err error) - clearError() getState() (folderState, time.Time, error) + setState(state folderState) + clearError() + setError(err error) } type Availability struct { @@ -1431,10 +1431,10 @@ func (m *Model) ScanFolders() map[string]error { } func (m *Model) ScanFolder(folder string) error { - return m.ScanFolderSubs(folder, nil) + return m.ScanFolderSubdirs(folder, nil) } -func (m *Model) ScanFolderSubs(folder string, subs []string) error { +func (m *Model) ScanFolderSubdirs(folder string, subs []string) error { m.fmut.Lock() runner, ok := m.folderRunners[folder] m.fmut.Unlock() @@ -1449,13 +1449,13 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error { return runner.Scan(subs) } -func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error { - for i, sub := range subs { +func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error { + for i, sub := range subDirs { sub = osutil.NativeFilename(sub) if p := filepath.Clean(filepath.Join(folder, sub)); !strings.HasPrefix(p, folder) { return errors.New("invalid subpath") } - subs[i] = sub + subDirs[i] = sub } m.fmut.Lock() @@ -1488,7 +1488,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error { // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. - subs = unifySubs(subs, func(f string) bool { + subDirs = unifySubs(subDirs, func(f string) bool { _, ok := fs.Get(protocol.LocalDeviceID, f) return ok }) @@ -1503,7 +1503,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error { fchan, err := scanner.Walk(scanner.Config{ Folder: folderCfg.ID, Dir: folderCfg.Path(), - Subs: subs, + Subs: subDirs, Matcher: ignores, BlockSize: protocol.BlockSize, TempNamer: defTempNamer, @@ -1556,15 +1556,15 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error { m.updateLocalsFromScanning(folder, batch) } - if len(subs) == 0 { + if len(subDirs) == 0 { // If we have no specific subdirectories to traverse, set it to one // empty prefix so we traverse the entire folder contents once. - subs = []string{""} + subDirs = []string{""} } // Do a scan of the database for each prefix, to check for deleted files. batch = batch[:0] - for _, sub := range subs { + for _, sub := range subDirs { var iterError error fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 0a8a47a9..7e335603 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -1402,7 +1402,7 @@ func TestIssue3028(t *testing.T) { os.Remove("testdata/testrm") os.Remove("testdata/testrm2") - m.ScanFolderSubs("default", []string{"testrm", "testrm2"}) + m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"}) // Verify that the number of files decreased by two and the number of // deleted files increases by two diff --git a/lib/model/rofolder.go b/lib/model/rofolder.go index 1f0d7bce..38a3842b 100644 --- a/lib/model/rofolder.go +++ b/lib/model/rofolder.go @@ -8,10 +8,8 @@ package model import ( "fmt" - "time" "github.com/syncthing/syncthing/lib/config" - "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/versioner" ) @@ -23,21 +21,13 @@ type roFolder struct { folder } -func newROFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner) service { +func newROFolder(model *Model, config config.FolderConfiguration, ver versioner.Versioner) service { return &roFolder{ folder: folder{ - stateTracker: stateTracker{ - folderID: cfg.ID, - mut: sync.NewMutex(), - }, - scan: folderscan{ - interval: time.Duration(cfg.RescanIntervalS) * time.Second, - timer: time.NewTimer(time.Millisecond), - now: make(chan rescanRequest), - delay: make(chan time.Duration), - }, - stop: make(chan struct{}), - model: model, + stateTracker: newStateTracker(config.ID), + scan: newFolderScanner(config), + stop: make(chan struct{}), + model: model, }, } } @@ -59,7 +49,7 @@ func (f *roFolder) Serve() { case <-f.scan.timer.C: if err := f.model.CheckFolderHealth(f.folderID); err != nil { l.Infoln("Skipping folder", f.folderID, "scan due to folder error:", err) - f.scan.reschedule() + f.scan.Reschedule() continue } @@ -71,7 +61,7 @@ func (f *roFolder) Serve() { // the same one as returned by CheckFolderHealth, though // duplicate set is handled by setError. f.setError(err) - f.scan.reschedule() + f.scan.Reschedule() continue } @@ -80,11 +70,11 @@ func (f *roFolder) Serve() { initialScanCompleted = true } - if f.scan.interval == 0 { + if f.scan.HasNoInterval() { continue } - f.scan.reschedule() + f.scan.Reschedule() case req := <-f.scan.now: req.err <- f.scanSubdirsIfHealthy(req.subdirs) diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 7be9bcd5..98a7d807 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -107,18 +107,10 @@ type rwFolder struct { func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner) service { f := &rwFolder{ folder: folder{ - stateTracker: stateTracker{ - folderID: cfg.ID, - mut: sync.NewMutex(), - }, - scan: folderscan{ - interval: time.Duration(cfg.RescanIntervalS) * time.Second, - timer: time.NewTimer(time.Millisecond), // The first scan should be done immediately. - now: make(chan rescanRequest), - delay: make(chan time.Duration), - }, - stop: make(chan struct{}), - model: model, + stateTracker: newStateTracker(cfg.ID), + scan: newFolderScanner(cfg), + stop: make(chan struct{}), + model: model, }, virtualMtimeRepo: db.NewVirtualMtimeRepo(model.db, cfg.ID), @@ -297,7 +289,7 @@ func (f *rwFolder) Serve() { // same time. case <-f.scan.timer.C: err := f.scanSubdirsIfHealthy(nil) - f.scan.reschedule() + f.scan.Reschedule() if err != nil { continue } diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index 1b781319..91c52f58 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -69,10 +69,8 @@ func setUpModel(file protocol.FileInfo) *Model { func setUpRwFolder(model *Model) rwFolder { return rwFolder{ folder: folder{ - stateTracker: stateTracker{ - folderID: "default", - }, - model: model, + stateTracker: newStateTracker("default"), + model: model, }, dir: "testdata", queue: newJobQueue(),