lib/model, lib/osutil: Verify target directory before pulling / requesting

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3798

lib/model/rwfolder.go

@@ -382,23 +382,81 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 	folderFiles := f.model.folderFiles[f.folderID]
 	f.model.fmut.RUnlock()
 
-	// !!!
-	// WithNeed takes a database snapshot (by necessity). By the time we've
-	// handled a bunch of files it might have become out of date and we might
-	// be attempting to sync with an old version of a file...
-	// !!!
-
 	changed := 0
+	var processDirectly []protocol.FileInfo
+
+	// Iterate the list of items that we need and sort them into piles.
+	// Regular files to pull goes into the file queue, everything else
+	// (directories, symlinks and deletes) goes into the "process directly"
+	// pile.
+
+	folderFiles.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+		if shouldIgnore(intf, ignores, f.ignoreDelete, defTempNamer) {
+			return true
+		}
+
+		if err := fileValid(intf); err != nil {
+			// The file isn't valid so we can't process it. Pretend that we
+			// tried and set the error for the file.
+			f.newError(intf.FileName(), err)
+			changed++
+			return true
+		}
+
+		file := intf.(protocol.FileInfo)
+
+		switch {
+		case file.IsDeleted():
+			processDirectly = append(processDirectly, file)
+			changed++
+
+		case file.Type == protocol.FileInfoTypeFile:
+			// Queue files for processing after directories and symlinks, if
+			// it has availability.
+
+			devices := folderFiles.Availability(file.Name)
+			for _, dev := range devices {
+				if f.model.ConnectedTo(dev) {
+					f.queue.Push(file.Name, file.Size, file.ModTime())
+					changed++
+					break
+				}
+			}
+
+		default:
+			// Directories, symlinks
+			processDirectly = append(processDirectly, file)
+			changed++
+		}
+
+		return true
+	})
+
+	// Sort the "process directly" pile by number of path components. This
+	// ensures that we handle parents before children.
+
+	sort.Sort(byComponentCount(processDirectly))
+
+	// Process the list.
 
 	fileDeletions := map[string]protocol.FileInfo{}
 	dirDeletions := []protocol.FileInfo{}
 	buckets := map[string][]protocol.FileInfo{}
 
-	handleItem := func(fi protocol.FileInfo) bool {
+	for _, fi := range processDirectly {
+		// Verify that the thing we are handling lives inside a directory,
+		// and not a symlink or empty space.
+		if !osutil.IsDir(f.dir, filepath.Dir(fi.Name)) {
+			f.newError(fi.Name, errNotDir)
+			continue
+		}
+
 		switch {
 		case fi.IsDeleted():
 			// A deleted file, directory or symlink
 			if fi.IsDirectory() {
+				// Perform directory deletions at the end, as we may have
+				// files to delete inside them before we get to that point.
 				dirDeletions = append(dirDeletions, fi)
 			} else {
 				fileDeletions[fi.Name] = fi
@@ -413,63 +471,22 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 					buckets[key] = append(buckets[key], df)
 				}
 			}
-			changed++
 
 		case fi.IsDirectory() && !fi.IsSymlink():
 			l.Debugln("Handling directory", fi.Name)
 			f.handleDir(fi)
-			changed++
 
 		case fi.IsSymlink():
 			l.Debugln("Handling symlink", fi.Name)
 			f.handleSymlink(fi)
-			changed++
 
 		default:
-			return false
+			l.Warnln(fi)
+			panic("unhandleable item type, can't happen")
 		}
-		return true
 	}
 
-	folderFiles.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
-		// Needed items are delivered sorted lexicographically. We'll handle
-		// directories as they come along, so parents before children. Files
-		// are queued and the order may be changed later.
-
-		if shouldIgnore(intf, ignores, f.ignoreDelete) {
-			return true
-		}
-
-		if err := fileValid(intf); err != nil {
-			// The file isn't valid so we can't process it. Pretend that we
-			// tried and set the error for the file.
-			f.newError(intf.FileName(), err)
-			changed++
-			return true
-		}
-
-		file := intf.(protocol.FileInfo)
-		l.Debugln(f, "handling", file.Name)
-		if !handleItem(file) {
-			// A new or changed file or symlink. This is the only case where
-			// we do stuff concurrently in the background. We only queue
-			// files where we are connected to at least one device that has
-			// the file.
-
-			devices := folderFiles.Availability(file.Name)
-			for _, dev := range devices {
-				if f.model.ConnectedTo(dev) {
-					f.queue.Push(file.Name, file.Size, file.ModTime())
-					changed++
-					break
-				}
-			}
-		}
-
-		return true
-	})
-
-	// Reorder the file queue according to configuration
+	// Now do the file queue. Reorder it according to configuration.
 
 	switch f.order {
 	case config.OrderRandom:
@@ -486,7 +503,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		f.queue.SortNewestFirst()
 	}
 
-	// Process the file queue
+	// Process the file queue.
 
 nextFile:
 	for {
@@ -509,10 +526,17 @@ nextFile:
 			continue
 		}
 
-		// Handles races where an index update arrives changing what the file
-		// is between queueing and retrieving it from the queue, effectively
-		// changing how the file should be handled.
-		if handleItem(fi) {
+		if fi.IsDeleted() || fi.Type != protocol.FileInfoTypeFile {
+			// The item has changed type or status in the index while we
+			// were processing directories above.
+			f.queue.Done(fileName)
+			continue
+		}
+
+		// Verify that the thing we are handling lives inside a directory,
+		// and not a symlink or empty space.
+		if !osutil.IsDir(f.dir, filepath.Dir(fi.Name)) {
+			f.newError(fi.Name, errNotDir)
 			continue
 		}
 
@@ -779,6 +803,7 @@ func (f *rwFolder) deleteDir(file protocol.FileInfo, matcher *ignore.Matcher) {
 		f.newError(file.Name, err)
 		return
 	}
+
 	// Delete any temporary files lying around in the directory
 	dir, _ := os.Open(realName)
 	if dir != nil {
@@ -1727,3 +1752,29 @@ func windowsInvalidFilename(name string) bool {
 	// The path must not contain any disallowed characters
 	return strings.ContainsAny(name, windowsDisallowedCharacters)
 }
+
+// byComponentCount sorts by the number of path components in Name, that is
+// "x/y" sorts before "foo/bar/baz".
+type byComponentCount []protocol.FileInfo
+
+func (l byComponentCount) Len() int {
+	return len(l)
+}
+
+func (l byComponentCount) Less(a, b int) bool {
+	return componentCount(l[a].Name) < componentCount(l[b].Name)
+}
+
+func (l byComponentCount) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func componentCount(name string) int {
+	count := 0
+	for _, codepoint := range name {
+		if codepoint == os.PathSeparator {
+			count++
+		}
+	}
+	return count
+}