all: Simultaneously walk fs and db on scan (fixes #2571, fixes #4573) (#4584)

When scanner.Walk detects a change, it now returns the new file info as well as the old file info. It also finds deleted and ignored files while scanning.
Also directory deletions are now always committed to db after their children to prevent temporary failure on remote due to non-empty directory.

lib/scanner/blockqueue.go

@@ -65,15 +65,15 @@ type parallelHasher struct {
 	fs            fs.Filesystem
 	blockSize     int
 	workers       int
-	outbox        chan<- protocol.FileInfo
-	inbox         <-chan protocol.FileInfo
+	outbox        chan<- ScanResult
+	inbox         <-chan ScanResult
 	counter       Counter
 	done          chan<- struct{}
 	useWeakHashes bool
 	wg            sync.WaitGroup
 }
 
-func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
+func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- ScanResult, inbox <-chan ScanResult, counter Counter, done chan<- struct{}, useWeakHashes bool) {
 	ph := &parallelHasher{
 		fs:            fs,
 		blockSize:     blockSize,
@@ -104,25 +104,25 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
 				return
 			}
 
-			if f.IsDirectory() || f.IsDeleted() {
+			if f.New.IsDirectory() || f.New.IsDeleted() {
 				panic("Bug. Asked to hash a directory or a deleted file.")
 			}
 
-			blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
+			blocks, err := HashFile(ctx, ph.fs, f.New.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
 			if err != nil {
-				l.Debugln("hash error:", f.Name, err)
+				l.Debugln("hash error:", f.New.Name, err)
 				continue
 			}
 
-			f.Blocks = blocks
+			f.New.Blocks = blocks
 
 			// The size we saw when initially deciding to hash the file
 			// might not have been the size it actually had when we hashed
 			// it. Update the size from the block list.
 
-			f.Size = 0
+			f.New.Size = 0
 			for _, b := range blocks {
-				f.Size += int64(b.Size)
+				f.New.Size += int64(b.Size)
 			}
 
 			select {