perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Convert folders to use filesystem abstraction

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
Audrius Butkevicius
2017-08-19 14:36:56 +00:00
committed by Jakob Borg
parent ab8c2fb5c7
commit 3d8b4a42b7
78 changed files with 2588 additions and 1665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
lib/ignore/ignore.go
View File

@@ -12,13 +12,13 @@ import (
"crypto/md5"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/gobwas/glob"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -77,6 +77,7 @@ type ChangeDetector interface {
}
type Matcher struct {
fs fs.Filesystem
lines []string // exact lines read from .stignore
patterns []Pattern // patterns including those from included files
withCache bool
@@ -105,8 +106,9 @@ func WithChangeDetector(cd ChangeDetector) Option {
}
}
func New(opts ...Option) *Matcher {
func New(fs fs.Filesystem, opts ...Option) *Matcher {
m := &Matcher{
fs: fs,
stop: make(chan struct{}),
mut: sync.NewMutex(),
}
@@ -114,7 +116,7 @@ func New(opts ...Option) *Matcher {
opt(m)
}
if m.changeDetector == nil {
m.changeDetector = newModtimeChecker()
m.changeDetector = newModtimeChecker(fs)
}
if m.withCache {
go m.clean(2 * time.Hour)
@@ -130,7 +132,7 @@ func (m *Matcher) Load(file string) error {
return nil
}
fd, err := os.Open(file)
fd, err := m.fs.Open(file)
if err != nil {
m.parseLocked(&bytes.Buffer{}, file)
return err
@@ -156,7 +158,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
}
func (m *Matcher) parseLocked(r io.Reader, file string) error {
lines, patterns, err := parseIgnoreFile(r, file, m.changeDetector)
lines, patterns, err := parseIgnoreFile(m.fs, r, file, m.changeDetector)
// Error is saved and returned at the end. We process the patterns
// (possibly blank) anyway.
@@ -298,12 +300,12 @@ func hashPatterns(patterns []Pattern) string {
return fmt.Sprintf("%x", h.Sum(nil))
}
func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error) {
func loadIgnoreFile(fs fs.Filesystem, file string, cd ChangeDetector) ([]string, []Pattern, error) {
if cd.Seen(file) {
return nil, nil, fmt.Errorf("multiple include of ignore file %q", file)
}
fd, err := os.Open(file)
fd, err := fs.Open(file)
if err != nil {
return nil, nil, err
}
@@ -316,10 +318,10 @@ func loadIgnoreFile(file string, cd ChangeDetector) ([]string, []Pattern, error)
cd.Remember(file, info.ModTime())
return parseIgnoreFile(fd, file, cd)
return parseIgnoreFile(fs, fd, file, cd)
}
func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd ChangeDetector) ([]string, []Pattern, error) {
var lines []string
var patterns []Pattern
@@ -386,7 +388,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
} else if strings.HasPrefix(line, "#include ") {
includeRel := line[len("#include "):]
includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
_, includePatterns, err := loadIgnoreFile(includeFile, cd)
_, includePatterns, err := loadIgnoreFile(fs, includeFile, cd)
if err != nil {
return fmt.Errorf("include of %q: %v", includeRel, err)
}
@@ -450,7 +452,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, cd ChangeDetector) ([]str
// path must be clean (i.e., in canonical shortest form).
func IsInternal(file string) bool {
internals := []string{".stfolder", ".stignore", ".stversions"}
pathSep := string(os.PathSeparator)
pathSep := string(fs.PathSeparator)
for _, internal := range internals {
if file == internal {
return true
@@ -463,8 +465,8 @@ func IsInternal(file string) bool {
}
// WriteIgnores is a convenience function to avoid code duplication
func WriteIgnores(path string, content []string) error {
fd, err := osutil.CreateAtomic(path)
func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error {
fd, err := osutil.CreateAtomicFilesystem(filesystem, path)
if err != nil {
return err
}
@@ -476,18 +478,20 @@ func WriteIgnores(path string, content []string) error {
if err := fd.Close(); err != nil {
return err
}
osutil.HideFile(path)
filesystem.Hide(path)
return nil
}
// modtimeChecker is the default implementation of ChangeDetector
type modtimeChecker struct {
fs fs.Filesystem
modtimes map[string]time.Time
}
func newModtimeChecker() *modtimeChecker {
func newModtimeChecker(fs fs.Filesystem) *modtimeChecker {
return &modtimeChecker{
fs: fs,
modtimes: map[string]time.Time{},
}
}
@@ -507,7 +511,7 @@ func (c *modtimeChecker) Reset() {
func (c *modtimeChecker) Changed() bool {
for name, modtime := range c.modtimes {
info, err := os.Stat(name)
info, err := c.fs.Stat(name)
if err != nil {
return true
}

114
lib/ignore/ignore_test.go
View File

@@ -15,11 +15,14 @@ import (
"runtime"
"testing"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func TestIgnore(t *testing.T) {
pats := New(WithCache(true))
err := pats.Load("testdata/.stignore")
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), WithCache(true))
err := pats.Load(".stignore")
if err != nil {
t.Fatal(err)
}
@@ -68,7 +71,7 @@ func TestExcludes(t *testing.T) {
i*2
!ign2
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -113,7 +116,7 @@ func TestFlagOrder(t *testing.T) {
(?i)(?d)(?d)!ign9
(?d)(?d)!ign10
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -148,7 +151,7 @@ func TestDeletables(t *testing.T) {
ign7
(?i)ign8
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -187,7 +190,7 @@ func TestBadPatterns(t *testing.T) {
}
for _, pat := range badPatterns {
err := New(WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
err := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true)).Parse(bytes.NewBufferString(pat), ".stignore")
if err == nil {
t.Errorf("No error for pattern %q", pat)
}
@@ -195,7 +198,7 @@ func TestBadPatterns(t *testing.T) {
}
func TestCaseSensitivity(t *testing.T) {
ign := New(WithCache(true))
ign := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := ign.Parse(bytes.NewBufferString("test"), ".stignore")
if err != nil {
t.Error(err)
@@ -225,29 +228,36 @@ func TestCaseSensitivity(t *testing.T) {
}
func TestCaching(t *testing.T) {
fd1, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
fd2, err := ioutil.TempFile("", "")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd1, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
fd2, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
defer fd1.Close()
defer fd2.Close()
defer os.Remove(fd1.Name())
defer os.Remove(fd2.Name())
defer fs.Remove(fd1.Name())
defer fs.Remove(fd2.Name())
_, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")
_, err = fd1.Write([]byte("/x/\n#include " + filepath.Base(fd2.Name()) + "\n"))
if err != nil {
t.Fatal(err)
}
fd2.WriteString("/y/\n")
fd2.Write([]byte("/y/\n"))
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd1.Name())
if err != nil {
t.Fatal(err)
@@ -280,10 +290,10 @@ func TestCaching(t *testing.T) {
// Modify the include file, expect empty cache. Ensure the timestamp on
// the file changes.
fd2.WriteString("/z/\n")
fd2.Write([]byte("/z/\n"))
fd2.Sync()
fakeTime := time.Now().Add(5 * time.Second)
os.Chtimes(fd2.Name(), fakeTime, fakeTime)
fs.Chtimes(fd2.Name(), fakeTime, fakeTime)
err = pats.Load(fd1.Name())
if err != nil {
@@ -312,10 +322,10 @@ func TestCaching(t *testing.T) {
// Modify the root file, expect cache to be invalidated
fd1.WriteString("/a/\n")
fd1.Write([]byte("/a/\n"))
fd1.Sync()
fakeTime = time.Now().Add(5 * time.Second)
os.Chtimes(fd1.Name(), fakeTime, fakeTime)
fs.Chtimes(fd1.Name(), fakeTime, fakeTime)
err = pats.Load(fd1.Name())
if err != nil {
@@ -354,7 +364,7 @@ func TestCommentsAndBlankLines(t *testing.T) {
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Error(err)
@@ -382,7 +392,7 @@ flamingo
*.crow
*.crow
`
pats := New()
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
b.Error(err)
@@ -411,20 +421,27 @@ flamingo
*.crow
`
// Caches per file, hence write the patterns to a file.
fd, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
b.Fatal(err)
}
_, err = fd.WriteString(stignore)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd, err := osutil.TempFile(fs, "", "")
if err != nil {
b.Fatal(err)
}
_, err = fd.Write([]byte(stignore))
defer fd.Close()
defer os.Remove(fd.Name())
defer fs.Remove(fd.Name())
if err != nil {
b.Fatal(err)
}
// Load the patterns
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
b.Fatal(err)
@@ -445,22 +462,29 @@ flamingo
}
func TestCacheReload(t *testing.T) {
fd, err := ioutil.TempFile("", "")
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
fd, err := osutil.TempFile(fs, "", "")
if err != nil {
t.Fatal(err)
}
defer fd.Close()
defer os.Remove(fd.Name())
defer fs.Remove(fd.Name())
// Ignore file matches f1 and f2
_, err = fd.WriteString("f1\nf2\n")
_, err = fd.Write([]byte("f1\nf2\n"))
if err != nil {
t.Fatal(err)
}
pats := New(WithCache(true))
pats := New(fs, WithCache(true))
err = pats.Load(fd.Name())
if err != nil {
t.Fatal(err)
@@ -488,13 +512,13 @@ func TestCacheReload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = fd.WriteString("f1\nf3\n")
_, err = fd.Write([]byte("f1\nf3\n"))
if err != nil {
t.Fatal(err)
}
fd.Sync()
fakeTime := time.Now().Add(5 * time.Second)
os.Chtimes(fd.Name(), fakeTime, fakeTime)
fs.Chtimes(fd.Name(), fakeTime, fakeTime)
err = pats.Load(fd.Name())
if err != nil {
@@ -515,7 +539,7 @@ func TestCacheReload(t *testing.T) {
}
func TestHash(t *testing.T) {
p1 := New(WithCache(true))
p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@@ -531,7 +555,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p2 := New(WithCache(true))
p2 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err = p2.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -546,7 +570,7 @@ func TestHash(t *testing.T) {
/ffile
lost+found
`
p3 := New(WithCache(true))
p3 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err = p3.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -570,7 +594,7 @@ func TestHash(t *testing.T) {
}
func TestHashOfEmpty(t *testing.T) {
p1 := New(WithCache(true))
p1 := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := p1.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@@ -608,7 +632,7 @@ func TestWindowsPatterns(t *testing.T) {
a/b
c\d
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -633,7 +657,7 @@ func TestAutomaticCaseInsensitivity(t *testing.T) {
A/B
c/d
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -652,7 +676,7 @@ func TestCommas(t *testing.T) {
foo,bar.txt
{baz,quux}.txt
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -683,7 +707,7 @@ func TestIssue3164(t *testing.T) {
(?d)(?i)/foo
(?d)(?i)**/bar
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -719,7 +743,7 @@ func TestIssue3174(t *testing.T) {
stignore := `
*ä*
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -734,7 +758,7 @@ func TestIssue3639(t *testing.T) {
stignore := `
foo/
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -767,7 +791,7 @@ func TestIssue3674(t *testing.T) {
{"as/dc", true},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -799,7 +823,7 @@ func TestGobwasGlobIssue18(t *testing.T) {
{"bbaa", false},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -859,7 +883,7 @@ func TestRoot(t *testing.T) {
{"b", true},
}
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)
@@ -876,12 +900,12 @@ func TestRoot(t *testing.T) {
func TestLines(t *testing.T) {
stignore := `
#include testdata/excludes
!/a
/*
`
pats := New(WithCache(true))
pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
if err != nil {
t.Fatal(err)