perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/fs, lib/model, lib/scanner: Make scans cancellable (fixes #3965)

Browse Source 
The folder already knew how to stop properly, but the fs.Walk() didn't
and can potentially take a very long time. This adds context support to
Walk and the underlying scanning stuff, and passes in an appropriate
context from above. The stop channel in model.folder is replaced with a
context for this purpose.

To test I added an infiniteFS that represents a large amount of data
(not actually infinite, but close) and verify that walking it is
properly stopped. For that to be implemented smoothly I moved out the
Walk function to it's own type, as typically the implementer of a new
filesystem type might not need or want to reimplement Walk.

It's somewhat tricky to test that this actually works properly on the
actual sendReceiveFolder and so on, as those are started from inside the
model and the filesystem isn't easily pluggable etc. Instead I've tested
that part manually by adding a huge folder and verifying that pause,
resume and reconfig do the right things by looking at debug output.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4117
This commit is contained in:
Jakob Borg
2017-04-26 00:15:23 +00:00
committed by Audrius Butkevicius
parent bdaef44765
commit d6fbfc3545
17 changed files with 282 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
lib/scanner/walk_test.go
View File

@@ -8,6 +8,7 @@ package scanner
import (
"bytes"
"context"
"crypto/rand"
"fmt"
"io"
@@ -59,7 +60,7 @@ func TestWalkSub(t *testing.T) {
t.Fatal(err)
}
fchan, err := Walk(Config{
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
Subs: []string{"dir2"},
BlockSize: 128 * 1024,
@@ -96,7 +97,7 @@ func TestWalk(t *testing.T) {
}
t.Log(ignores)
fchan, err := Walk(Config{
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
BlockSize: 128 * 1024,
Matcher: ignores,
@@ -120,7 +121,7 @@ func TestWalk(t *testing.T) {
}
func TestWalkError(t *testing.T) {
_, err := Walk(Config{
_, err := Walk(context.TODO(), Config{
Dir: "testdata-missing",
BlockSize: 128 * 1024,
Hashers: 2,
@@ -130,7 +131,7 @@ func TestWalkError(t *testing.T) {
t.Error("no error from missing directory")
}
_, err = Walk(Config{
_, err = Walk(context.TODO(), Config{
Dir: "testdata/bar",
BlockSize: 128 * 1024,
})
@@ -148,7 +149,7 @@ func TestVerify(t *testing.T) {
progress := newByteCounter()
defer progress.Close()
blocks, err := Blocks(buf, blocksize, -1, progress, false)
blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress, false)
if err != nil {
t.Fatal(err)
}
@@ -276,7 +277,7 @@ func TestNormalization(t *testing.T) {
func TestIssue1507(t *testing.T) {
w := &walker{}
c := make(chan protocol.FileInfo, 100)
fn := w.walkAndHashFiles(c, c)
fn := w.walkAndHashFiles(context.TODO(), c, c)
fn("", nil, protocol.ErrClosed)
}
@@ -297,7 +298,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
// Scan it
fchan, err := Walk(Config{
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
})
@@ -342,7 +343,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
// Scan it
fchan, err := Walk(Config{
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
})
@@ -364,7 +365,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
}
func walkDir(dir string) ([]protocol.FileInfo, error) {
fchan, err := Walk(Config{
fchan, err := Walk(context.TODO(), Config{
Dir: dir,
BlockSize: 128 * 1024,
AutoNormalize: true,
@@ -434,7 +435,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@@ -458,3 +459,68 @@ func initTestFile() {
panic(err)
}
}
func TestStopWalk(t *testing.T) {
// Create tree that is 100 levels deep, with each level containing 100
// files (each 1 MB) and 100 directories (in turn containing 100 files
// and 100 directories, etc). That is, in total > 100^100 files and as
// many directories. It'll take a while to scan, giving us time to
// cancel it and make sure the scan stops.
fs := fs.NewWalkFilesystem(&infiniteFS{100, 100, 1e6})
const numHashers = 4
ctx, cancel := context.WithCancel(context.Background())
fchan, err := Walk(ctx, Config{
Dir: "testdir",
BlockSize: 128 * 1024,
Hashers: numHashers,
Filesystem: fs,
ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
})
if err != nil {
t.Fatal(err)
}
// Receive a few entries to make sure the walker is up and running,
// scanning both files and dirs. Do some quick sanity tests on the
// returned file entries to make sure we are not just reading crap from
// a closed channel or something.
dirs := 0
files := 0
for {
f := <-fchan
t.Log("Scanned", f)
if f.IsDirectory() {
if len(f.Name) == 0 || f.Permissions == 0 {
t.Error("Bad directory entry", f)
}
dirs++
} else {
if len(f.Name) == 0 || len(f.Blocks) == 0 || f.Permissions == 0 {
t.Error("Bad file entry", f)
}
files++
}
if dirs > 5 && files > 5 {
break
}
}
// Cancel the walker.
cancel()
// Empty out any waiting entries and wait for the channel to close.
// Count them, they should be zero or very few - essentially, each
// hasher has the choice of returning a fully handled entry or
// cancelling, but they should not start on another item.
extra := 0
for range fchan {
extra++
}
t.Log("Extra entries:", extra)
if extra > numHashers {
t.Error("unexpected extra entries received after cancel")
}
}