Rework .stignore functionality (fixes #561) (...)

- Only one .stignore is supported, at the repo root
 - Negative patterns (!) are supported
 - Ignore patterns affect sent and received indexes, not only scanning

scanner/testdata/.stignore

@@ -1,5 +1,4 @@
 #include excludes
-#include nonexistent-file
 
 bfile
 dir1/cfile

scanner/testdata/excludes

@@ -1,4 +1,2 @@
 dir2/dfile
-#include excludes
 #include further-excludes
-#include loop-excludes

scanner/testdata/loop-excludes

@@ -1 +0,0 @@
-#include excludes

scanner/walk.go

@@ -5,19 +5,14 @@
 package scanner
 
 import (
-	"bufio"
 	"errors"
-	"io"
 	"os"
-	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
-	"strings"
 
 	"code.google.com/p/go.text/unicode/norm"
 
-	"github.com/syncthing/syncthing/fnmatch"
+	"github.com/syncthing/syncthing/ignore"
 	"github.com/syncthing/syncthing/lamport"
 	"github.com/syncthing/syncthing/protocol"
 )
@@ -29,8 +24,8 @@ type Walker struct {
 	Sub string
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
-	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
-	IgnoreFile string
+	// List of patterns to ignore
+	Ignores ignore.Patterns
 	// If TempNamer is not nil, it is used to ignore tempory files when walking.
 	TempNamer TempNamer
 	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
@@ -57,7 +52,7 @@ type CurrentFiler interface {
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	if debug {
-		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.IgnoreFile)
+		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.Ignores)
 	}
 
 	err := checkDir(w.Dir)
@@ -69,11 +64,8 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	hashedFiles := make(chan protocol.FileInfo)
 	newParallelHasher(w.Dir, w.BlockSize, runtime.NumCPU(), hashedFiles, files)
 
-	var ignores []*regexp.Regexp
 	go func() {
-		filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, &ignores))
-
-		hashFiles := w.walkAndHashFiles(files, ignores)
+		hashFiles := w.walkAndHashFiles(files)
 		filepath.Walk(filepath.Join(w.Dir, w.Sub), hashFiles)
 		close(files)
 	}()
@@ -86,113 +78,7 @@ func (w *Walker) CleanTempFiles() {
 	filepath.Walk(w.Dir, w.cleanTempFile)
 }
 
-func (w *Walker) loadIgnoreFiles(dir string, ignores *[]*regexp.Regexp) filepath.WalkFunc {
-	return func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			return nil
-		}
-
-		rn, err := filepath.Rel(dir, p)
-		if err != nil {
-			return nil
-		}
-
-		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
-			pn := filepath.Clean(pn)
-			filesSeen := make(map[string]map[string]bool)
-			dirIgnores := loadIgnoreFile(p, pn, filesSeen)
-			*ignores = append(*ignores, dirIgnores...)
-		}
-
-		return nil
-	}
-}
-
-func loadIgnoreFile(ignFile, base string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
-	fd, err := os.Open(ignFile)
-	if err != nil {
-		return nil
-	}
-	defer fd.Close()
-	return parseIgnoreFile(fd, base, ignFile, filesSeen)
-}
-
-func parseIgnoreFile(fd io.Reader, base, currentFile string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
-	var exps []*regexp.Regexp
-	scanner := bufio.NewScanner(fd)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" {
-			continue
-		}
-
-		if strings.HasPrefix(line, "/") {
-			// Pattern is rooted in the current dir only
-			exp, err := fnmatch.Convert(path.Join(base, line[1:]), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		} else if strings.HasPrefix(line, "**/") {
-			// Add the pattern as is, and without **/ so it matches in current dir
-			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-
-			exp, err = fnmatch.Convert(path.Join(base, line[3:]), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		} else if strings.HasPrefix(line, "#include ") {
-			includeFile := filepath.Join(filepath.Dir(currentFile), strings.Replace(line, "#include ", "", 1))
-			if _, err := os.Stat(includeFile); os.IsNotExist(err) {
-				l.Infoln("Could not open ignore include file", includeFile)
-			} else {
-				seen := false
-				if seenByCurrent, ok := filesSeen[currentFile]; ok {
-					_, seen = seenByCurrent[includeFile]
-				}
-
-				if seen {
-					l.Warnf("Recursion detected while including %s from %s", includeFile, currentFile)
-				} else {
-					if filesSeen[currentFile] == nil {
-						filesSeen[currentFile] = make(map[string]bool)
-					}
-					filesSeen[currentFile][includeFile] = true
-					includes := loadIgnoreFile(includeFile, base, filesSeen)
-					exps = append(exps, includes...)
-				}
-			}
-		} else {
-			// Path name or pattern, add it so it matches files both in
-			// current directory and subdirs.
-			exp, err := fnmatch.Convert(path.Join(base, line), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-
-			exp, err = fnmatch.Convert(path.Join(base, "**", line), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		}
-	}
-
-	return exps
-}
-
-func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ignores []*regexp.Regexp) filepath.WalkFunc {
+func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
@@ -221,7 +107,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ignores []*regex
 			return nil
 		}
 
-		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ignores, rn) {
+		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" || w.Ignores.Match(rn) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)
@@ -305,18 +191,6 @@ func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
 	return nil
 }
 
-func (w *Walker) ignoreFile(patterns []*regexp.Regexp, file string) bool {
-	for _, pattern := range patterns {
-		if pattern.MatchString(file) {
-			if debug {
-				l.Debugf("%q matches %v", file, pattern)
-			}
-			return true
-		}
-	}
-	return false
-}
-
 func checkDir(dir string) error {
 	if info, err := os.Lstat(dir); err != nil {
 		return err

scanner/walk_test.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/syncthing/syncthing/ignore"
 	"github.com/syncthing/syncthing/protocol"
 )
 
@@ -30,9 +31,8 @@ var testdata = testfileList{
 	{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
 	{"dir2", 128, ""},
 	{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
-	{"excludes", 78, "1f5ac95d9e6fb2516629a029d788d27953c7bb2f4dc09184b660fdda0c8f2f04"},
+	{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
 	{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
-	{"loop-excludes", 18, "2db057aa82a8b8fe4b1367ccc875259ed4b8020255820d4e3d4bfe78f0dd3f2a"},
 }
 
 var correctIgnores = map[string][]string{
@@ -47,11 +47,16 @@ func init() {
 }
 
 func TestWalkSub(t *testing.T) {
+	ignores, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	w := Walker{
-		Dir:        "testdata",
-		Sub:        "dir2",
-		BlockSize:  128 * 1024,
-		IgnoreFile: ".stignore",
+		Dir:       "testdata",
+		Sub:       "dir2",
+		BlockSize: 128 * 1024,
+		Ignores:   ignores,
 	}
 	fchan, err := w.Walk()
 	var files []protocol.FileInfo
@@ -77,10 +82,16 @@ func TestWalkSub(t *testing.T) {
 }
 
 func TestWalk(t *testing.T) {
+	ignores, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(ignores)
+
 	w := Walker{
-		Dir:        "testdata",
-		BlockSize:  128 * 1024,
-		IgnoreFile: ".stignore",
+		Dir:       "testdata",
+		BlockSize: 128 * 1024,
+		Ignores:   ignores,
 	}
 
 	fchan, err := w.Walk()
@@ -102,9 +113,8 @@ func TestWalk(t *testing.T) {
 
 func TestWalkError(t *testing.T) {
 	w := Walker{
-		Dir:        "testdata-missing",
-		BlockSize:  128 * 1024,
-		IgnoreFile: ".stignore",
+		Dir:       "testdata-missing",
+		BlockSize: 128 * 1024,
 	}
 	_, err := w.Walk()
 
@@ -113,9 +123,8 @@ func TestWalkError(t *testing.T) {
 	}
 
 	w = Walker{
-		Dir:        "testdata/bar",
-		BlockSize:  128 * 1024,
-		IgnoreFile: ".stignore",
+		Dir:       "testdata/bar",
+		BlockSize: 128 * 1024,
 	}
 	_, err = w.Walk()
 
@@ -124,67 +133,6 @@ func TestWalkError(t *testing.T) {
 	}
 }
 
-func TestIgnore(t *testing.T) {
-	patStr := bytes.NewBufferString(`
-		t2
-		/t3
-		sub/dir/*
-		*/other/test
-		**/deep
-	`)
-	patterns := parseIgnoreFile(patStr, "", "", make(map[string]map[string]bool))
-
-	patStr = bytes.NewBufferString(`
-		bar
-		z*
-		q[abc]x
-	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo", "", make(map[string]map[string]bool))...)
-
-	patStr = bytes.NewBufferString(`
-		quux
-		.*
-	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo/baz", "", make(map[string]map[string]bool))...)
-
-	var tests = []struct {
-		f string
-		r bool
-	}{
-		{filepath.Join("foo", "bar"), true},
-		{filepath.Join("t3"), true},
-		{filepath.Join("foofoo"), false},
-		{filepath.Join("foo", "quux"), false},
-		{filepath.Join("foo", "zuux"), true},
-		{filepath.Join("foo", "qzuux"), false},
-		{filepath.Join("foo", "baz", "t1"), false},
-		{filepath.Join("foo", "baz", "t2"), true},
-		{filepath.Join("foo", "baz", "t3"), false},
-		{filepath.Join("foo", "baz", "bar"), true},
-		{filepath.Join("foo", "baz", "quuxa"), false},
-		{filepath.Join("foo", "baz", "aquux"), false},
-		{filepath.Join("foo", "baz", ".quux"), true},
-		{filepath.Join("foo", "baz", "zquux"), true},
-		{filepath.Join("foo", "baz", "quux"), true},
-		{filepath.Join("foo", "bazz", "quux"), false},
-		{filepath.Join("sub", "dir", "hej"), true},
-		{filepath.Join("deeper", "sub", "dir", "hej"), true},
-		{filepath.Join("other", "test"), false},
-		{filepath.Join("sub", "other", "test"), true},
-		{filepath.Join("deeper", "sub", "other", "test"), true},
-		{filepath.Join("deep"), true},
-		{filepath.Join("deeper", "deep"), true},
-		{filepath.Join("deeper", "deeper", "deep"), true},
-	}
-
-	w := Walker{}
-	for i, tc := range tests {
-		if r := w.ignoreFile(patterns, tc.f); r != tc.r {
-			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
-		}
-	}
-}
-
 type fileList []protocol.FileInfo
 
 func (f fileList) Len() int {