perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model, gui: Allow creating and editing ignores of paused folders (fixes #3608)

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3996
LGTM: calmh, AudriusButkevicius
This commit is contained in:
Simon Frei
2017-04-01 09:58:06 +00:00
committed by Audrius Butkevicius
parent c5e0c47989
commit 25b314f5f1
12 changed files with 202 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
lib/ignore/ignore.go
View File

@@ -19,6 +19,7 @@ import (
"time"
"github.com/gobwas/glob"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -64,6 +65,7 @@ func (r Result) IsCaseFolded() bool {
}
type Matcher struct {
lines []string
patterns []Pattern
withCache bool
matches *cache
@@ -120,7 +122,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
}
func (m *Matcher) parseLocked(r io.Reader, file string) error {
patterns, err := parseIgnoreFile(r, file, m.modtimes)
lines, patterns, err := parseIgnoreFile(r, file, m.modtimes)
// Error is saved and returned at the end. We process the patterns
// (possibly blank) anyway.
@@ -131,6 +133,7 @@ func (m *Matcher) parseLocked(r io.Reader, file string) error {
}
m.curHash = newHash
m.lines = lines
m.patterns = patterns
if m.withCache {
m.matches = newCache(patterns)
@@ -206,6 +209,13 @@ func (m *Matcher) Match(file string) (result Result) {
return resultNotMatched
}
// Lines return a list of the unprocessed lines in .stignore at last load
func (m *Matcher) Lines() []string {
m.mut.Lock()
defer m.mut.Unlock()
return m.lines
}
// Patterns return a list of the loaded patterns, as they've been parsed
func (m *Matcher) Patterns() []string {
if m == nil {
@@ -274,27 +284,28 @@ func hashPatterns(patterns []Pattern) string {
return fmt.Sprintf("%x", h.Sum(nil))
}
func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]Pattern, error) {
func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]string, []Pattern, error) {
if _, ok := modtimes[file]; ok {
return nil, fmt.Errorf("Multiple include of ignore file %q", file)
return nil, nil, fmt.Errorf("multiple include of ignore file %q", file)
}
fd, err := os.Open(file)
if err != nil {
return nil, err
return nil, nil, err
}
defer fd.Close()
info, err := fd.Stat()
if err != nil {
return nil, err
return nil, nil, err
}
modtimes[file] = info.ModTime()
return parseIgnoreFile(fd, file, modtimes)
}
func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]Pattern, error) {
func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]string, []Pattern, error) {
var lines []string
var patterns []Pattern
defaultResult := resultInclude
@@ -360,11 +371,12 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.
} else if strings.HasPrefix(line, "#include ") {
includeRel := line[len("#include "):]
includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
includes, err := loadIgnoreFile(includeFile, modtimes)
includeLines, includePatterns, err := loadIgnoreFile(includeFile, modtimes)
if err != nil {
return fmt.Errorf("include of %q: %v", includeRel, err)
}
patterns = append(patterns, includes...)
lines = append(lines, includeLines...)
patterns = append(patterns, includePatterns...)
} else {
// Path name or pattern, add it so it matches files both in
// current directory and subdirs.
@@ -389,6 +401,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.
var err error
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
lines = append(lines, line)
switch {
case line == "":
continue
@@ -411,11 +424,11 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.
}
}
if err != nil {
return nil, err
return nil, nil, err
}
}
return patterns, nil
return lines, patterns, nil
}
// IsInternal returns true if the file, as a path relative to the folder
@@ -434,3 +447,22 @@ func IsInternal(file string) bool {
}
return false
}
// WriteIgnores is a convenience function to avoid code duplication
func WriteIgnores(path string, content []string) error {
fd, err := osutil.CreateAtomic(path)
if err != nil {
return err
}
for _, line := range content {
fmt.Fprintln(fd, line)
}
if err := fd.Close(); err != nil {
return err
}
osutil.HideFile(path)
return nil
}

9
lib/model/folder.go
View File

@@ -10,9 +10,11 @@ import "time"
type folder struct {
stateTracker
scan folderScanner
model *Model
stop chan struct{}
scan folderScanner
model *Model
stop chan struct{}
initialScanCompleted chan struct{}
}
func (f *folder) IndexUpdated() {
@@ -23,6 +25,7 @@ func (f *folder) DelayScan(next time.Duration) {
}
func (f *folder) Scan(subdirs []string) error {
<-f.initialScanCompleted
return f.scan.Scan(subdirs)
}
func (f *folder) Stop() {

76
lib/model/model.go
View File

@@ -7,7 +7,6 @@
package model
import (
"bufio"
"crypto/tls"
"encoding/json"
"errors"
@@ -1252,66 +1251,51 @@ func (m *Model) ConnectedTo(deviceID protocol.DeviceID) bool {
}
func (m *Model) GetIgnores(folder string) ([]string, []string, error) {
var lines []string
m.fmut.RLock()
cfg, ok := m.folderCfgs[folder]
m.fmut.RUnlock()
if !ok {
return lines, nil, fmt.Errorf("Folder %s does not exist", folder)
}
if !cfg.HasMarker() {
return lines, nil, fmt.Errorf("Folder %s stopped", folder)
}
fd, err := os.Open(filepath.Join(cfg.Path(), ".stignore"))
if err != nil {
if os.IsNotExist(err) {
return lines, nil, nil
if ok {
if !cfg.HasMarker() {
return nil, nil, fmt.Errorf("Folder %s stopped", folder)
}
l.Warnln("Loading .stignore:", err)
return lines, nil, err
}
defer fd.Close()
scanner := bufio.NewScanner(fd)
for scanner.Scan() {
lines = append(lines, strings.TrimSpace(scanner.Text()))
m.fmut.RLock()
ignores := m.folderIgnores[folder]
m.fmut.RUnlock()
return ignores.Lines(), ignores.Patterns(), nil
}
m.fmut.RLock()
patterns := m.folderIgnores[folder].Patterns()
m.fmut.RUnlock()
if cfg, ok := m.cfg.Folders()[folder]; ok {
matcher := ignore.New(false)
path := filepath.Join(cfg.Path(), ".stignore")
if err := matcher.Load(path); err != nil {
return nil, nil, err
}
return matcher.Lines(), matcher.Patterns(), nil
}
return lines, patterns, nil
return nil, nil, fmt.Errorf("Folder %s does not exist", folder)
}
func (m *Model) SetIgnores(folder string, content []string) error {
cfg, ok := m.folderCfgs[folder]
cfg, ok := m.cfg.Folders()[folder]
if !ok {
return fmt.Errorf("Folder %s does not exist", folder)
}
path := filepath.Join(cfg.Path(), ".stignore")
fd, err := osutil.CreateAtomic(path)
if err != nil {
if err := ignore.WriteIgnores(filepath.Join(cfg.Path(), ".stignore"), content); err != nil {
l.Warnln("Saving .stignore:", err)
return err
}
for _, line := range content {
fmt.Fprintln(fd, line)
m.fmut.RLock()
runner, ok := m.folderRunners[folder]
m.fmut.RUnlock()
if ok {
return runner.Scan(nil)
}
if err := fd.Close(); err != nil {
l.Warnln("Saving .stignore:", err)
return err
}
osutil.HideFile(path)
return m.ScanFolder(folder)
return nil
}
// OnHello is called when an device connects to us.
@@ -2395,9 +2379,13 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
for folderID, cfg := range toFolders {
if _, ok := fromFolders[folderID]; !ok {
// A folder was added.
l.Debugln(m, "adding folder", folderID)
m.AddFolder(cfg)
m.StartFolder(folderID)
if cfg.Paused {
l.Infoln(m, "Paused folder", cfg.Description())
} else {
l.Infoln(m, "Adding folder", cfg.Description())
m.AddFolder(cfg)
m.StartFolder(folderID)
}
}
}

65
lib/model/model_test.go
View File

@@ -927,7 +927,7 @@ func TestIntroducer(t *testing.T) {
}
}
func TestIgnores(t *testing.T) {
func changeIgnores(t *testing.T, m *Model, expected []string) {
arrEqual := func(a, b []string) bool {
if len(a) != len(b) {
return false
@@ -941,22 +941,6 @@ func TestIgnores(t *testing.T) {
return true
}
// Assure a clean start state
ioutil.WriteFile("testdata/.stfolder", nil, 0644)
ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
m.AddFolder(defaultFolderConfig)
m.StartFolder("default")
m.ServeBackground()
defer m.Stop()
expected := []string{
".*",
"quux",
}
ignores, _, err := m.GetIgnores("default")
if err != nil {
t.Error(err)
@@ -999,8 +983,34 @@ func TestIgnores(t *testing.T) {
if !arrEqual(ignores, expected) {
t.Errorf("Incorrect ignores: %v != %v", ignores, expected)
}
}
_, _, err = m.GetIgnores("doesnotexist")
func TestIgnores(t *testing.T) {
// Assure a clean start state
ioutil.WriteFile("testdata/.stfolder", nil, 0644)
ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
m.ServeBackground()
defer m.Stop()
// m.cfg.SetFolder is not usable as it is non-blocking, and there is no
// way to know when the folder is actually added.
m.AddFolder(defaultFolderConfig)
m.StartFolder("default")
// Make sure the initial scan has finished (ScanFolders is blocking)
m.ScanFolders()
expected := []string{
".*",
"quux",
}
changeIgnores(t, m, expected)
_, _, err := m.GetIgnores("doesnotexist")
if err == nil {
t.Error("No error")
}
@@ -1016,6 +1026,16 @@ func TestIgnores(t *testing.T) {
if err == nil {
t.Error("No error")
}
// Repeat tests with paused folder
pausedDefaultFolderConfig := defaultFolderConfig
pausedDefaultFolderConfig.Paused = true
m.RestartFolder(pausedDefaultFolderConfig)
// Here folder initialization is not an issue as a paused folder isn't
// added to the model and thus there is no initial scan happening.
changeIgnores(t, m, expected)
}
func TestROScanRecovery(t *testing.T) {
@@ -1763,13 +1783,8 @@ func TestIssue3028(t *testing.T) {
m.StartFolder("default")
m.ServeBackground()
// Ugly hack for testing: reach into the model for the SendReceiveFolder and wait
// for it to complete the initial scan. The risk is that it otherwise
// runs during our modifications and screws up the test.
m.fmut.RLock()
folder := m.folderRunners["default"].(*sendReceiveFolder)
m.fmut.RUnlock()
<-folder.initialScanCompleted
// Make sure the initial scan has finished (ScanFolders is blocking)
m.ScanFolders()
// Get a count of how many files are there now

16
lib/model/rofolder.go
View File

@@ -26,10 +26,11 @@ type sendOnlyFolder struct {
func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ *fs.MtimeFS) service {
return &sendOnlyFolder{
folder: folder{
stateTracker: newStateTracker(cfg.ID),
scan: newFolderScanner(cfg),
stop: make(chan struct{}),
model: model,
stateTracker: newStateTracker(cfg.ID),
scan: newFolderScanner(cfg),
stop: make(chan struct{}),
model: model,
initialScanCompleted: make(chan struct{}),
},
FolderConfiguration: cfg,
}
@@ -43,7 +44,6 @@ func (f *sendOnlyFolder) Serve() {
f.scan.timer.Stop()
}()
initialScanCompleted := false
for {
select {
case <-f.stop:
@@ -68,9 +68,11 @@ func (f *sendOnlyFolder) Serve() {
continue
}
if !initialScanCompleted {
select {
case <-f.initialScanCompleted:
default:
l.Infoln("Completed initial scan (ro) of", f.Description())
initialScanCompleted = true
close(f.initialScanCompleted)
}
if f.scan.HasNoInterval() {

15
lib/model/rwfolder.go
View File

@@ -96,17 +96,16 @@ type sendReceiveFolder struct {
errors map[string]string // path -> error string
errorsMut sync.Mutex
initialScanCompleted chan (struct{}) // exposed for testing
}
func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service {
f := &sendReceiveFolder{
folder: folder{
stateTracker: newStateTracker(cfg.ID),
scan: newFolderScanner(cfg),
stop: make(chan struct{}),
model: model,
stateTracker: newStateTracker(cfg.ID),
scan: newFolderScanner(cfg),
stop: make(chan struct{}),
model: model,
initialScanCompleted: make(chan struct{}),
},
FolderConfiguration: cfg,
@@ -119,8 +118,6 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
errorsMut: sync.NewMutex(),
initialScanCompleted: make(chan struct{}),
}
f.configureCopiersAndPullers()
@@ -1063,7 +1060,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
// sweep is complete. As we do retries, we'll queue the scan
// for this file up to ten times, but the last nine of those
// scans will be cheap...
go f.scan.Scan([]string{file.Name})
go f.Scan([]string{file.Name})
return
}
}

14
lib/model/rwfolder_test.go
View File

@@ -77,11 +77,12 @@ func setUpModel(file protocol.FileInfo) *Model {
return model
}
func setUpSendReceiveFolder(model *Model) sendReceiveFolder {
return sendReceiveFolder{
func setUpSendReceiveFolder(model *Model) *sendReceiveFolder {
f := &sendReceiveFolder{
folder: folder{
stateTracker: newStateTracker("default"),
model: model,
stateTracker: newStateTracker("default"),
model: model,
initialScanCompleted: make(chan struct{}),
},
mtimeFS: fs.NewMtimeFS(fs.DefaultFilesystem, db.NewNamespacedKV(model.db, "mtime")),
@@ -90,6 +91,11 @@ func setUpSendReceiveFolder(model *Model) sendReceiveFolder {
errors: make(map[string]string),
errorsMut: sync.NewMutex(),
}
// Folders are never actually started, so no initial scan will be done
close(f.initialScanCompleted)
return f
}
// Layout of the files: (indexes from the above array)