lib/ignore: Replace lib/fnmatch with github.com/gobwas/glob
Because it's literally ten times faster: benchmark old ns/op new ns/op delta BenchmarkMatch-8 13842 1200 -91.33% BenchmarkMatchCached-8 139 147 +5.76% benchmark old allocs new allocs delta BenchmarkMatch-8 0 0 +0.00% BenchmarkMatchCached-8 0 0 +0.00% benchmark old bytes new bytes delta BenchmarkMatch-8 12 0 -100.00% BenchmarkMatchCached-8 0 0 +0.00%
This commit is contained in:
Jakob Borg
committed by
Audrius Butkevicius
Audrius Butkevicius
parent
46e913dc23
commit
4c3cd4c9e3
@@ -1,79 +0,0 @@
|
||||
// Copyright (C) 2014 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package fnmatch
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
NoEscape = (1 << iota)
|
||||
PathName
|
||||
CaseFold
|
||||
)
|
||||
|
||||
func Convert(pattern string, flags int) (*regexp.Regexp, error) {
|
||||
any := "."
|
||||
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
flags |= NoEscape | CaseFold
|
||||
pattern = filepath.FromSlash(pattern)
|
||||
if flags&PathName != 0 {
|
||||
any = `[^\\]`
|
||||
}
|
||||
case "darwin":
|
||||
flags |= CaseFold
|
||||
fallthrough
|
||||
default:
|
||||
if flags&PathName != 0 {
|
||||
any = `[^/]`
|
||||
}
|
||||
}
|
||||
|
||||
if flags&NoEscape != 0 {
|
||||
pattern = strings.Replace(pattern, `\`, `\\`, -1)
|
||||
} else {
|
||||
pattern = strings.Replace(pattern, `\*`, "[:escapedstar:]", -1)
|
||||
pattern = strings.Replace(pattern, `\?`, "[:escapedques:]", -1)
|
||||
pattern = strings.Replace(pattern, `\.`, "[:escapeddot:]", -1)
|
||||
}
|
||||
|
||||
// Characters that are special in regexps but not in glob, must be
|
||||
// escaped.
|
||||
for _, char := range []string{`.`, `+`, `$`, `^`, `(`, `)`, `|`} {
|
||||
pattern = strings.Replace(pattern, char, `\`+char, -1)
|
||||
}
|
||||
|
||||
pattern = strings.Replace(pattern, `**`, `[:doublestar:]`, -1)
|
||||
pattern = strings.Replace(pattern, `*`, any+`*`, -1)
|
||||
pattern = strings.Replace(pattern, `[:doublestar:]`, `.*`, -1)
|
||||
pattern = strings.Replace(pattern, `?`, any, -1)
|
||||
|
||||
pattern = strings.Replace(pattern, `[:escapedstar:]`, `\*`, -1)
|
||||
pattern = strings.Replace(pattern, `[:escapedques:]`, `\?`, -1)
|
||||
pattern = strings.Replace(pattern, `[:escapeddot:]`, `\.`, -1)
|
||||
|
||||
pattern = `^` + pattern + `$`
|
||||
if flags&CaseFold != 0 {
|
||||
pattern = `(?i)` + pattern
|
||||
}
|
||||
return regexp.Compile(pattern)
|
||||
}
|
||||
|
||||
// Match matches the pattern against the string, with the given flags, and
|
||||
// returns true if the match is successful.
|
||||
func Match(pattern, s string, flags int) (bool, error) {
|
||||
exp, err := Convert(pattern, flags)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exp.MatchString(s), nil
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright (C) 2014 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package fnmatch
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testcase struct {
|
||||
pat string
|
||||
name string
|
||||
flags int
|
||||
match bool
|
||||
}
|
||||
|
||||
var testcases = []testcase{
|
||||
{"", "", 0, true},
|
||||
{"*", "", 0, true},
|
||||
{"*", "foo", 0, true},
|
||||
{"*", "bar", 0, true},
|
||||
{"*", "*", 0, true},
|
||||
{"**", "f", 0, true},
|
||||
{"**", "foo.txt", 0, true},
|
||||
{"*.*", "foo.txt", 0, true},
|
||||
{"foo*.txt", "foobar.txt", 0, true},
|
||||
{"foo.txt", "foo.txt", 0, true},
|
||||
|
||||
{"foo.txt", "bar/foo.txt", 0, false},
|
||||
{"*/foo.txt", "bar/foo.txt", 0, true},
|
||||
{"f?o.txt", "foo.txt", 0, true},
|
||||
{"f?o.txt", "fooo.txt", 0, false},
|
||||
{"f[ab]o.txt", "foo.txt", 0, false},
|
||||
{"f[ab]o.txt", "fao.txt", 0, true},
|
||||
{"f[ab]o.txt", "fbo.txt", 0, true},
|
||||
{"f[ab]o.txt", "fco.txt", 0, false},
|
||||
{"f[ab]o.txt", "fabo.txt", 0, false},
|
||||
{"f[ab]o.txt", "f[ab]o.txt", 0, false},
|
||||
{"f\\[ab\\]o.txt", "f[ab]o.txt", NoEscape, false},
|
||||
|
||||
{"*foo.txt", "bar/foo.txt", 0, true},
|
||||
{"*foo.txt", "bar/foo.txt", PathName, false},
|
||||
{"*/foo.txt", "bar/foo.txt", 0, true},
|
||||
{"*/foo.txt", "bar/foo.txt", PathName, true},
|
||||
{"*/foo.txt", "bar/baz/foo.txt", 0, true},
|
||||
{"*/foo.txt", "bar/baz/foo.txt", PathName, false},
|
||||
{"**/foo.txt", "bar/baz/foo.txt", 0, true},
|
||||
{"**/foo.txt", "bar/baz/foo.txt", PathName, true},
|
||||
|
||||
{"foo.txt", "foo.TXT", CaseFold, true},
|
||||
|
||||
// These characters are literals in glob, but not in regexp.
|
||||
{"hey$hello", "hey$hello", 0, true},
|
||||
{"hey^hello", "hey^hello", 0, true},
|
||||
{"hey{hello", "hey{hello", 0, true},
|
||||
{"hey}hello", "hey}hello", 0, true},
|
||||
{"hey(hello", "hey(hello", 0, true},
|
||||
{"hey)hello", "hey)hello", 0, true},
|
||||
{"hey|hello", "hey|hello", 0, true},
|
||||
{"hey|hello", "hey|other", 0, false},
|
||||
}
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
|
||||
case "darwin":
|
||||
testcases = append(testcases, testcase{"foo.txt", "foo.TXT", 0, true})
|
||||
fallthrough
|
||||
default:
|
||||
testcases = append(testcases, testcase{"f\\[ab\\]o.txt", "f[ab]o.txt", 0, true})
|
||||
testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", 0, true})
|
||||
testcases = append(testcases, testcase{"foo\\*.txt", "foo*.txt", 0, true})
|
||||
testcases = append(testcases, testcase{"foo\\.txt", "foo.txt", NoEscape, false})
|
||||
testcases = append(testcases, testcase{"f\\\\\\[ab\\\\\\]o.txt", "f\\[ab\\]o.txt", 0, true})
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
if m, err := Match(tc.pat, filepath.FromSlash(tc.name), tc.flags); m != tc.match {
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
t.Errorf("Match(%q, %q, %d) != %v", tc.pat, tc.name, tc.flags, tc.match)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalid(t *testing.T) {
|
||||
if _, err := Match("foo[bar", "...", 0); err == nil {
|
||||
t.Error("Unexpected nil error")
|
||||
}
|
||||
}
|
||||
@@ -14,24 +14,30 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fnmatch"
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
|
||||
type Pattern struct {
|
||||
match *regexp.Regexp
|
||||
include bool
|
||||
pattern string
|
||||
match glob.Glob
|
||||
include bool
|
||||
foldCase bool
|
||||
}
|
||||
|
||||
func (p Pattern) String() string {
|
||||
if p.include {
|
||||
return p.match.String()
|
||||
ret := p.pattern
|
||||
if !p.include {
|
||||
ret = "!" + ret
|
||||
}
|
||||
return "(?exclude)" + p.match.String()
|
||||
if p.foldCase {
|
||||
ret = "(?i)" + ret
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
type Matcher struct {
|
||||
@@ -119,9 +125,20 @@ func (m *Matcher) Match(file string) (result bool) {
|
||||
}
|
||||
|
||||
// Check all the patterns for a match.
|
||||
file = filepath.ToSlash(file)
|
||||
var lowercaseFile string
|
||||
for _, pattern := range m.patterns {
|
||||
if pattern.match.MatchString(file) {
|
||||
return pattern.include
|
||||
if pattern.foldCase {
|
||||
if lowercaseFile == "" {
|
||||
lowercaseFile = strings.ToLower(file)
|
||||
}
|
||||
if pattern.match.Match(lowercaseFile) {
|
||||
return pattern.include
|
||||
}
|
||||
} else {
|
||||
if pattern.match.Match(file) {
|
||||
return pattern.include
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +146,7 @@ func (m *Matcher) Match(file string) (result bool) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Patterns return a list of the loaded regexp patterns, as strings
|
||||
// Patterns return a list of the loaded patterns, as they've been parsed
|
||||
func (m *Matcher) Patterns() []string {
|
||||
if m == nil {
|
||||
return nil
|
||||
@@ -200,38 +217,43 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
|
||||
var patterns []Pattern
|
||||
|
||||
addPattern := func(line string) error {
|
||||
include := true
|
||||
pattern := Pattern{
|
||||
pattern: line,
|
||||
include: true,
|
||||
foldCase: runtime.GOOS == "darwin" || runtime.GOOS == "windows",
|
||||
}
|
||||
|
||||
if strings.HasPrefix(line, "!") {
|
||||
line = line[1:]
|
||||
include = false
|
||||
pattern.include = false
|
||||
}
|
||||
|
||||
flags := fnmatch.PathName
|
||||
if strings.HasPrefix(line, "(?i)") {
|
||||
line = line[4:]
|
||||
flags |= fnmatch.CaseFold
|
||||
line = strings.ToLower(line[4:])
|
||||
pattern.foldCase = true
|
||||
}
|
||||
|
||||
var err error
|
||||
if strings.HasPrefix(line, "/") {
|
||||
// Pattern is rooted in the current dir only
|
||||
exp, err := fnmatch.Convert(line[1:], flags)
|
||||
pattern.match, err = glob.Compile(line[1:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pattern %q in ignore file", line)
|
||||
}
|
||||
patterns = append(patterns, Pattern{exp, include})
|
||||
patterns = append(patterns, pattern)
|
||||
} else if strings.HasPrefix(line, "**/") {
|
||||
// Add the pattern as is, and without **/ so it matches in current dir
|
||||
exp, err := fnmatch.Convert(line, flags)
|
||||
pattern.match, err = glob.Compile(line)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pattern %q in ignore file", line)
|
||||
}
|
||||
patterns = append(patterns, Pattern{exp, include})
|
||||
patterns = append(patterns, pattern)
|
||||
|
||||
exp, err = fnmatch.Convert(line[3:], flags)
|
||||
pattern.match, err = glob.Compile(line[3:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pattern %q in ignore file", line)
|
||||
}
|
||||
patterns = append(patterns, Pattern{exp, include})
|
||||
patterns = append(patterns, pattern)
|
||||
} else if strings.HasPrefix(line, "#include ") {
|
||||
includeRel := line[len("#include "):]
|
||||
includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
|
||||
@@ -243,17 +265,17 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
|
||||
} else {
|
||||
// Path name or pattern, add it so it matches files both in
|
||||
// current directory and subdirs.
|
||||
exp, err := fnmatch.Convert(line, flags)
|
||||
pattern.match, err = glob.Compile(line)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pattern %q in ignore file", line)
|
||||
}
|
||||
patterns = append(patterns, Pattern{exp, include})
|
||||
patterns = append(patterns, pattern)
|
||||
|
||||
exp, err = fnmatch.Convert("**/"+line, flags)
|
||||
pattern.match, err = glob.Compile("**/" + line)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid pattern %q in ignore file", line)
|
||||
}
|
||||
patterns = append(patterns, Pattern{exp, include})
|
||||
patterns = append(patterns, pattern)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -267,6 +289,10 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
|
||||
continue
|
||||
case strings.HasPrefix(line, "//"):
|
||||
continue
|
||||
}
|
||||
|
||||
line = filepath.ToSlash(line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "#"):
|
||||
err = addPattern(line)
|
||||
case strings.HasSuffix(line, "/**"):
|
||||
|
||||
@@ -509,3 +509,29 @@ func TestHashOfEmpty(t *testing.T) {
|
||||
t.Error("there are more than zero patterns")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWindowsPatterns(t *testing.T) {
|
||||
// We should accept patterns as both a/b and a\b and match that against
|
||||
// both kinds of slash as well.
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("Windows specific test")
|
||||
return
|
||||
}
|
||||
|
||||
stignore := `
|
||||
a/b
|
||||
c\d
|
||||
`
|
||||
pats := New(true)
|
||||
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tests := []string{`a\b`, `c\d`}
|
||||
for _, pat := range tests {
|
||||
if !pats.Match(pat) {
|
||||
t.Errorf("Should match %s", pat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user