Add scan percentages (fixes #1030)

lib/scanner/walk.go

@@ -12,11 +12,13 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"time"
 	"unicode/utf8"
 
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/symlinks"
@@ -39,6 +41,8 @@ func init() {
 }
 
 type Walker struct {
+	// Folder for which the walker has been created
+	Folder string
 	// Dir is the base directory for the walk
 	Dir string
 	// Limit walking to these paths within Dir, or no limit if Sub is empty
@@ -66,6 +70,9 @@ type Walker struct {
 	Hashers int
 	// Our vector clock id
 	ShortID uint64
+	// Optional progress tick interval which defines how often FolderScanProgress
+	// events are emitted. Negative number means disabled.
+	ProgressTickIntervalS int
 }
 
 type TempNamer interface {
@@ -92,12 +99,13 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 		return nil, err
 	}
 
-	files := make(chan protocol.FileInfo)
-	hashedFiles := make(chan protocol.FileInfo)
-	newParallelHasher(w.Dir, w.BlockSize, w.Hashers, hashedFiles, files)
+	toHashChan := make(chan protocol.FileInfo)
+	finishedChan := make(chan protocol.FileInfo)
 
+	// A routine which walks the filesystem tree, and sends files which have
+	// been modified to the counter routine.
 	go func() {
-		hashFiles := w.walkAndHashFiles(files, hashedFiles)
+		hashFiles := w.walkAndHashFiles(toHashChan, finishedChan)
 		if len(w.Subs) == 0 {
 			filepath.Walk(w.Dir, hashFiles)
 		} else {
@@ -105,10 +113,77 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 				filepath.Walk(filepath.Join(w.Dir, sub), hashFiles)
 			}
 		}
-		close(files)
+		close(toHashChan)
 	}()
 
-	return hashedFiles, nil
+	// We're not required to emit scan progress events, just kick off hashers,
+	// and feed inputs directly from the walker.
+	if w.ProgressTickIntervalS < 0 {
+		newParallelHasher(w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil)
+		return finishedChan, nil
+	}
+
+	// Defaults to every 2 seconds.
+	if w.ProgressTickIntervalS == 0 {
+		w.ProgressTickIntervalS = 2
+	}
+
+	ticker := time.NewTicker(time.Duration(w.ProgressTickIntervalS) * time.Second)
+
+	// We need to emit progress events, hence we create a routine which buffers
+	// the list of files to be hashed, counts the total number of
+	// bytes to hash, and once no more files need to be hashed (chan gets closed),
+	// start a routine which periodically emits FolderScanProgress events,
+	// until a stop signal is sent by the parallel hasher.
+	// Parallel hasher is stopped by this routine when we close the channel over
+	// which it receives the files we ask it to hash.
+	go func() {
+		var filesToHash []protocol.FileInfo
+		var total, progress uint64
+		for file := range toHashChan {
+			filesToHash = append(filesToHash, file)
+			total += uint64(file.CachedSize)
+		}
+
+		realToHashChan := make(chan protocol.FileInfo)
+		done := make(chan struct{})
+		newParallelHasher(w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, &progress, done)
+
+		// A routine which actually emits the FolderScanProgress events
+		// every w.ProgressTicker ticks, until the hasher routines terminate.
+		go func() {
+			for {
+				select {
+				case <-done:
+					if debug {
+						l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher)
+					}
+					ticker.Stop()
+					return
+				case <-ticker.C:
+					current := atomic.LoadUint64(&progress)
+					if debug {
+						l.Debugf("Walk %s %s current progress %d/%d (%d%%)", w.Dir, w.Subs, current, total, current*100/total)
+					}
+					events.Default.Log(events.FolderScanProgress, map[string]interface{}{
+						"folder":  w.Folder,
+						"current": current,
+						"total":   total,
+					})
+				}
+			}
+		}()
+
+		for _, file := range filesToHash {
+			if debug {
+				l.Debugln("real to hash:", file.Name)
+			}
+			realToHashChan <- file
+		}
+		close(realToHashChan)
+	}()
+
+	return finishedChan, nil
 }
 
 func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.WalkFunc {
@@ -241,7 +316,7 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 				return skip
 			}
 
-			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
+			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0, nil)
 			if err != nil {
 				if debug {
 					l.Debugln("hash link error:", p, err)
@@ -272,10 +347,10 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 			}
 
 			if debug {
-				l.Debugln("symlink to hash:", p, f)
+				l.Debugln("symlink changedb:", p, f)
 			}
 
-			fchan <- f
+			dchan <- f
 
 			return skip
 		}
@@ -349,10 +424,11 @@ func (w *Walker) walkAndHashFiles(fchan, dchan chan protocol.FileInfo) filepath.
 			}
 
 			f := protocol.FileInfo{
-				Name:     rn,
-				Version:  cf.Version.Update(w.ShortID),
-				Flags:    flags,
-				Modified: mtime.Unix(),
+				Name:       rn,
+				Version:    cf.Version.Update(w.ShortID),
+				Flags:      flags,
+				Modified:   mtime.Unix(),
+				CachedSize: info.Size(),
 			}
 			if debug {
 				l.Debugln("to hash:", p, f)