perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Add invalid/ignored files to global list, announce to peers (fixes #623)

Browse Source 
This lets us determine accurate completion status for remote peers when they
have ignored files.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4460
This commit is contained in:
Simon Frei
2017-11-11 19:18:17 +00:00
committed by Jakob Borg
parent ec4c3bae0d
commit c080f677cb
17 changed files with 446 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
lib/model/rwfolder.go
View File

@@ -69,6 +69,7 @@ const (
dbUpdateDeleteFile
dbUpdateShortcutFile
dbUpdateHandleSymlink
dbUpdateInvalidate
)
const (
@@ -234,7 +235,9 @@ func (f *sendReceiveFolder) pull(prevSeq int64, prevIgnoreHash string) (curSeq i
f.model.fmut.RUnlock()
curSeq = prevSeq
if curIgnoreHash = curIgnores.Hash(); curIgnoreHash != prevIgnoreHash {
curIgnoreHash = curIgnores.Hash()
ignoresChanged := curIgnoreHash != prevIgnoreHash
if ignoresChanged {
// The ignore patterns have changed. We need to re-evaluate if
// there are files we need now that were ignored before.
l.Debugln(f, "ignore patterns have changed, resetting curSeq")
@@ -263,7 +266,7 @@ func (f *sendReceiveFolder) pull(prevSeq int64, prevIgnoreHash string) (curSeq i
for {
tries++
changed = f.pullerIteration(curIgnores)
changed := f.pullerIteration(curIgnores, ignoresChanged)
l.Debugln(f, "changed", changed)
if changed == 0 {
@@ -317,7 +320,7 @@ func (f *sendReceiveFolder) pull(prevSeq int64, prevIgnoreHash string) (curSeq i
// returns the number items that should have been synced (even those that
// might have failed). One puller iteration handles all files currently
// flagged as needed in the folder.
func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher, ignoresChanged bool) int {
pullChan := make(chan pullBlockState)
copyChan := make(chan copyBlocksState)
finisherChan := make(chan *sharedPullerState)
@@ -374,15 +377,21 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
if shouldIgnore(intf, ignores, f.IgnoreDelete) {
// Don't iterate over invalid/ignored files unless ignores have changed
iterate := folderFiles.WithNeed
if ignoresChanged {
iterate = folderFiles.WithNeedOrInvalid
}
iterate(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
if f.IgnoreDelete && intf.IsDeleted() {
return true
}
if err := fileValid(intf); err != nil {
// The file isn't valid so we can't process it. Pretend that we
// tried and set the error for the file.
f.newError("need", intf.FileName(), err)
// If filename isn't valid, we can terminate early with an appropriate error.
// in case it is deleted, we don't care about the filename, so don't complain.
if !intf.IsDeleted() && runtime.GOOS == "windows" && fs.WindowsInvalidFilename(intf.FileName()) {
f.newError("need", intf.FileName(), fs.ErrInvalidFilename)
changed++
return true
}
@@ -390,6 +399,11 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
file := intf.(protocol.FileInfo)
switch {
case ignores.ShouldIgnore(file.Name):
file.Invalidate(f.model.id.Short())
l.Debugln(f, "Handling ignored file", file)
f.dbUpdates <- dbUpdateJob{file, dbUpdateInvalidate}
case file.IsDeleted():
processDirectly = append(processDirectly, file)
changed++
@@ -403,9 +417,15 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
if f.model.ConnectedTo(dev) {
f.queue.Push(file.Name, file.Size, file.ModTime())
changed++
break
return true
}
}
l.Debugln(f, "Needed file is unavailable", file)
case runtime.GOOS == "windows" && file.IsSymlink():
file.Invalidate(f.model.id.Short())
l.Debugln(f, "Invalidating symlink (unsupported)", file.Name)
f.dbUpdates <- dbUpdateJob{file, dbUpdateInvalidate}
default:
// Directories, symlinks
@@ -449,7 +469,7 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
// number, hence the deletion coming in again as part of
// WithNeed, furthermore, the file can simply be of the wrong
// type if we haven't yet managed to pull it.
if ok && !df.IsDeleted() && !df.IsSymlink() && !df.IsDirectory() {
if ok && !df.IsDeleted() && !df.IsSymlink() && !df.IsDirectory() && !df.IsInvalid() {
// Put files into buckets per first hash
key := string(df.Blocks[0].Hash)
buckets[key] = append(buckets[key], df)
@@ -457,11 +477,11 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int {
}
case fi.IsDirectory() && !fi.IsSymlink():
l.Debugln("Handling directory", fi.Name)
l.Debugln(f, "Handling directory", fi.Name)
f.handleDir(fi)
case fi.IsSymlink():
l.Debugln("Handling symlink", fi.Name)
l.Debugln(f, "Handling symlink", fi.Name)
f.handleSymlink(fi)
default:
@@ -566,13 +586,13 @@ nextFile:
doneWg.Wait()
for _, file := range fileDeletions {
l.Debugln("Deleting file", file.Name)
l.Debugln(f, "Deleting file", file.Name)
f.deleteFile(file)
}
for i := range dirDeletions {
dir := dirDeletions[len(dirDeletions)-i-1]
l.Debugln("Deleting dir", dir.Name)
l.Debugln(f, "Deleting dir", dir.Name)
f.deleteDir(dir, ignores)
}
@@ -1516,8 +1536,9 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() {
changedDirs[filepath.Dir(job.file.Name)] = struct{}{}
case dbUpdateHandleDir:
changedDirs[job.file.Name] = struct{}{}
case dbUpdateHandleSymlink:
// fsyncing symlinks is only supported by MacOS, ignore
case dbUpdateHandleSymlink, dbUpdateInvalidate:
// fsyncing symlinks is only supported by MacOS
// and invalidated files are db only changes -> no sync
}
if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) {
@@ -1722,47 +1743,6 @@ func (l fileErrorList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
// fileValid returns nil when the file is valid for processing, or an error if it's not
func fileValid(file db.FileIntf) error {
switch {
case file.IsDeleted():
// We don't care about file validity if we're not supposed to have it
return nil
case runtime.GOOS == "windows" && file.IsSymlink():
return errSymlinksUnsupported
case runtime.GOOS == "windows" && windowsInvalidFilename(file.FileName()):
return fs.ErrInvalidFilename
}
return nil
}
var windowsDisallowedCharacters = string([]rune{
'<', '>', ':', '"', '|', '?', '*',
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
31,
})
func windowsInvalidFilename(name string) bool {
// None of the path components should end in space
for _, part := range strings.Split(name, `\`) {
if len(part) == 0 {
continue
}
if part[len(part)-1] == ' ' {
// Names ending in space are not valid.
return true
}
}
// The path must not contain any disallowed characters
return strings.ContainsAny(name, windowsDisallowedCharacters)
}
// byComponentCount sorts by the number of path components in Name, that is
// "x/y" sorts before "foo/bar/baz".
type byComponentCount []protocol.FileInfo