Rework .stignore functionality (fixes #561) (...)

- Only one .stignore is supported, at the repo root
 - Negative patterns (!) are supported
 - Ignore patterns affect sent and received indexes, not only scanning

ignore/ignore.go

@@ -0,0 +1,146 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package ignore
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/syncthing/syncthing/fnmatch"
+)
+
+type Pattern struct {
+	match   *regexp.Regexp
+	include bool
+}
+
+type Patterns []Pattern
+
+func Load(file string) (Patterns, error) {
+	seen := make(map[string]bool)
+	return loadIgnoreFile(file, seen)
+}
+
+func Parse(r io.Reader, file string) (Patterns, error) {
+	seen := map[string]bool{
+		file: true,
+	}
+	return parseIgnoreFile(r, file, seen)
+}
+
+func (l Patterns) Match(file string) bool {
+	for _, pattern := range l {
+		if pattern.match.MatchString(file) {
+			return pattern.include
+		}
+	}
+	return false
+}
+
+func loadIgnoreFile(file string, seen map[string]bool) (Patterns, error) {
+	if seen[file] {
+		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
+	}
+	seen[file] = true
+
+	fd, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	return parseIgnoreFile(fd, file, seen)
+}
+
+func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Patterns, error) {
+	var exps Patterns
+
+	addPattern := func(line string) error {
+		include := true
+		if strings.HasPrefix(line, "!") {
+			line = line[1:]
+			include = false
+		}
+
+		if strings.HasPrefix(line, "/") {
+			// Pattern is rooted in the current dir only
+			exp, err := fnmatch.Convert(line[1:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "**/") {
+			// Add the pattern as is, and without **/ so it matches in current dir
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert(line[3:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "#include ") {
+			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
+			includes, err := loadIgnoreFile(includeFile, seen)
+			if err != nil {
+				return err
+			} else {
+				exps = append(exps, includes...)
+			}
+		} else {
+			// Path name or pattern, add it so it matches files both in
+			// current directory and subdirs.
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert("**/"+line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		}
+		return nil
+	}
+
+	scanner := bufio.NewScanner(fd)
+	var err error
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case line == "":
+			continue
+		case strings.HasPrefix(line, "#"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/**"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/"):
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "**")
+			}
+		default:
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "/**")
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return exps, nil
+}

ignore/ignore_test.go

@@ -0,0 +1,104 @@
+package ignore_test
+
+import (
+	"bytes"
+	"path/filepath"
+	"testing"
+
+	"github.com/syncthing/syncthing/ignore"
+)
+
+func TestIgnore(t *testing.T) {
+	pats, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"afile", false},
+		{"bfile", true},
+		{"cfile", false},
+		{"dfile", false},
+		{"efile", true},
+		{"ffile", true},
+
+		{"dir1", false},
+		{filepath.Join("dir1", "cfile"), true},
+		{filepath.Join("dir1", "dfile"), false},
+		{filepath.Join("dir1", "efile"), true},
+		{filepath.Join("dir1", "ffile"), false},
+
+		{"dir2", false},
+		{filepath.Join("dir2", "cfile"), false},
+		{filepath.Join("dir2", "dfile"), true},
+		{filepath.Join("dir2", "efile"), true},
+		{filepath.Join("dir2", "ffile"), false},
+
+		{filepath.Join("dir3"), true},
+		{filepath.Join("dir3", "afile"), true},
+	}
+
+	for i, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
+		}
+	}
+}
+
+func TestExcludes(t *testing.T) {
+	stignore := `
+	!iex2
+	!ign1/ex
+	ign1
+	i*2
+	!ign2
+	`
+	pats, err := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"ign1", true},
+		{"ign2", true},
+		{"ibla2", true},
+		{"iex2", false},
+		{"ign1/ign", true},
+		{"ign1/ex", false},
+		{"ign1/iex2", false},
+		{"iex2/ign", false},
+		{"foo/bar/ign1", true},
+		{"foo/bar/ign2", true},
+		{"foo/bar/iex2", false},
+	}
+
+	for _, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect match for %s: %v != %v", tc.f, r, tc.r)
+		}
+	}
+}
+
+func TestBadPatterns(t *testing.T) {
+	var badPatterns = []string{
+		"[",
+		"/[",
+		"**/[",
+		"#include nonexistent",
+		"#include .stignore",
+		"!#include makesnosense",
+	}
+
+	for _, pat := range badPatterns {
+		parsed, err := ignore.Parse(bytes.NewBufferString(pat), ".stignore")
+		if err == nil {
+			t.Errorf("No error for pattern %q: %v", pat, parsed)
+		}
+	}
+}

ignore/testdata/.stignore

@@ -0,0 +1,6 @@
+#include excludes
+
+bfile
+dir1/cfile
+**/efile
+/ffile

ignore/testdata/dir3/cfile

@@ -0,0 +1 @@
+baz

ignore/testdata/dir3/dfile

@@ -0,0 +1 @@
+quux

ignore/testdata/excludes

@@ -0,0 +1,2 @@
+dir2/dfile
+#include further-excludes

ignore/testdata/further-excludes

@@ -0,0 +1 @@
+dir3