perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Convert folders to use filesystem abstraction

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
Audrius Butkevicius
2017-08-19 14:36:56 +00:00
committed by Jakob Borg
parent ab8c2fb5c7
commit 3d8b4a42b7
78 changed files with 2588 additions and 1665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib/scanner/blockqueue.go
View File

@@ -9,7 +9,6 @@ package scanner
import (
"context"
"errors"
"path/filepath"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
@@ -64,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
// is closed and all items handled.
type parallelHasher struct {
fs fs.Filesystem
dir string
blockSize int
workers int
outbox chan<- protocol.FileInfo
@@ -75,10 +73,9 @@ type parallelHasher struct {
wg sync.WaitGroup
}
func newParallelHasher(ctx context.Context, fs fs.Filesystem, dir string, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
ph := &parallelHasher{
fs: fs,
dir: dir,
blockSize: blockSize,
workers: workers,
outbox: outbox,
@@ -111,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
panic("Bug. Asked to hash a directory or a deleted file.")
}
blocks, err := HashFile(ctx, ph.fs, filepath.Join(ph.dir, f.Name), ph.blockSize, ph.counter, ph.useWeakHashes)
blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
if err != nil {
l.Debugln("hash error:", f.Name, err)
continue

29
lib/scanner/infinitefs_test.go
View File

@@ -19,6 +19,7 @@ import (
)
type infiniteFS struct {
fs.Filesystem
width int // number of files and directories per level
depth int // number of tree levels to simulate
filesize int64 // size of each file in bytes
@@ -50,18 +51,6 @@ func (i infiniteFS) Open(name string) (fs.File, error) {
return &fakeFile{name, i.filesize, 0}, nil
}
func (infiniteFS) Chmod(name string, mode fs.FileMode) error { return errNotSupp }
func (infiniteFS) Chtimes(name string, atime time.Time, mtime time.Time) error { return errNotSupp }
func (infiniteFS) Create(name string) (fs.File, error) { return nil, errNotSupp }
func (infiniteFS) CreateSymlink(name, target string) error { return errNotSupp }
func (infiniteFS) Mkdir(name string, perm fs.FileMode) error { return errNotSupp }
func (infiniteFS) ReadSymlink(name string) (string, error) { return "", errNotSupp }
func (infiniteFS) Remove(name string) error { return errNotSupp }
func (infiniteFS) Rename(oldname, newname string) error { return errNotSupp }
func (infiniteFS) Stat(name string) (fs.FileInfo, error) { return nil, errNotSupp }
func (infiniteFS) SymlinksSupported() bool { return false }
func (infiniteFS) Walk(root string, walkFn fs.WalkFunc) error { return errNotSupp }
type fakeInfo struct {
name string
size int64
@@ -71,7 +60,7 @@ func (f fakeInfo) Name() string { return f.name }
func (f fakeInfo) Mode() fs.FileMode { return 0755 }
func (f fakeInfo) Size() int64 { return f.size }
func (f fakeInfo) ModTime() time.Time { return time.Unix(1234567890, 0) }
func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") }
func (f fakeInfo) IsDir() bool { return strings.Contains(filepath.Base(f.name), "dir") || f.name == "." }
func (f fakeInfo) IsRegular() bool { return !f.IsDir() }
func (f fakeInfo) IsSymlink() bool { return false }
@@ -81,6 +70,10 @@ type fakeFile struct {
readOffset int64
}
func (f *fakeFile) Name() string {
return f.name
}
func (f *fakeFile) Read(bs []byte) (int, error) {
remaining := f.size - f.readOffset
if remaining == 0 {
@@ -98,6 +91,10 @@ func (f *fakeFile) Stat() (fs.FileInfo, error) {
return fakeInfo{f.name, f.size}, nil
}
func (f *fakeFile) WriteAt(bs []byte, offs int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Close() error { return nil }
func (f *fakeFile) Truncate(size int64) error { return errNotSupp }
func (f *fakeFile) Write([]byte) (int, error) { return 0, errNotSupp }
func (f *fakeFile) WriteAt([]byte, int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Close() error { return nil }
func (f *fakeFile) Truncate(size int64) error { return errNotSupp }
func (f *fakeFile) ReadAt([]byte, int64) (int, error) { return 0, errNotSupp }
func (f *fakeFile) Seek(int64, int) (int64, error) { return 0, errNotSupp }
func (f *fakeFile) Sync() error { return nil }

106
lib/scanner/walk.go
View File

@@ -9,7 +9,6 @@ package scanner
import (
"context"
"errors"
"path/filepath"
"runtime"
"sync/atomic"
"time"
@@ -42,8 +41,6 @@ func init() {
type Config struct {
// Folder for which the walker has been created
Folder string
// Dir is the base directory for the walk
Dir string
// Limit walking to these paths within Dir, or no limit if Sub is empty
Subs []string
// BlockSize controls the size of the block used when hashing.
@@ -86,7 +83,7 @@ func Walk(ctx context.Context, cfg Config) (chan protocol.FileInfo, error) {
w.CurrentFiler = noCurrentFiler{}
}
if w.Filesystem == nil {
w.Filesystem = fs.DefaultFilesystem
panic("no filesystem specified")
}
return w.walk(ctx)
@@ -99,7 +96,7 @@ type walker struct {
// Walk returns the list of files found in the local folder by scanning the
// file system. Files are blockwise hashed.
func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher)
if err := w.checkDir(); err != nil {
return nil, err
@@ -113,10 +110,10 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
go func() {
hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
if len(w.Subs) == 0 {
w.Filesystem.Walk(w.Dir, hashFiles)
w.Filesystem.Walk(".", hashFiles)
} else {
for _, sub := range w.Subs {
w.Filesystem.Walk(filepath.Join(w.Dir, sub), hashFiles)
w.Filesystem.Walk(sub, hashFiles)
}
}
close(toHashChan)
@@ -125,7 +122,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
// We're not required to emit scan progress events, just kick off hashers,
// and feed inputs directly from the walker.
if w.ProgressTickIntervalS < 0 {
newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
return finishedChan, nil
}
@@ -156,7 +153,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
done := make(chan struct{})
progress := newByteCounter()
newParallelHasher(ctx, w.Filesystem, w.Dir, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
// A routine which actually emits the FolderScanProgress events
// every w.ProgressTicker ticks, until the hasher routines terminate.
@@ -166,13 +163,13 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
for {
select {
case <-done:
l.Debugln("Walk progress done", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher)
ticker.Stop()
return
case <-ticker.C:
current := progress.Total()
rate := progress.Rate()
l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Dir, w.Subs, current, total, rate/1024/1024, current*100/total)
l.Debugf("Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
events.Default.Log(events.FolderScanProgress, map[string]interface{}{
"folder": w.Folder,
"current": current,
@@ -203,7 +200,7 @@ func (w *walker) walk(ctx context.Context) (chan protocol.FileInfo, error) {
func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protocol.FileInfo) fs.WalkFunc {
now := time.Now()
return func(absPath string, info fs.FileInfo, err error) error {
return func(path string, info fs.FileInfo, err error) error {
select {
case <-ctx.Done():
return ctx.Err()
@@ -219,58 +216,52 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
}
if err != nil {
l.Debugln("error:", absPath, info, err)
l.Debugln("error:", path, info, err)
return skip
}
relPath, err := filepath.Rel(w.Dir, absPath)
if err != nil {
l.Debugln("rel error:", absPath, err)
return skip
}
if relPath == "." {
if path == "." {
return nil
}
info, err = w.Filesystem.Lstat(absPath)
info, err = w.Filesystem.Lstat(path)
// An error here would be weird as we've already gotten to this point, but act on it nonetheless
if err != nil {
return skip
}
if ignore.IsTemporary(relPath) {
l.Debugln("temporary:", relPath)
if ignore.IsTemporary(path) {
l.Debugln("temporary:", path)
if info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
w.Filesystem.Remove(absPath)
l.Debugln("removing temporary:", relPath, info.ModTime())
w.Filesystem.Remove(path)
l.Debugln("removing temporary:", path, info.ModTime())
}
return nil
}
if ignore.IsInternal(relPath) {
l.Debugln("ignored (internal):", relPath)
if ignore.IsInternal(path) {
l.Debugln("ignored (internal):", path)
return skip
}
if w.Matcher.Match(relPath).IsIgnored() {
l.Debugln("ignored (patterns):", relPath)
if w.Matcher.Match(path).IsIgnored() {
l.Debugln("ignored (patterns):", path)
return skip
}
if !utf8.ValidString(relPath) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", relPath)
if !utf8.ValidString(path) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", path)
return skip
}
relPath, shouldSkip := w.normalizePath(absPath, relPath)
path, shouldSkip := w.normalizePath(path)
if shouldSkip {
return skip
}
switch {
case info.IsSymlink():
if err := w.walkSymlink(ctx, absPath, relPath, dchan); err != nil {
if err := w.walkSymlink(ctx, path, dchan); err != nil {
return err
}
if info.IsDir() {
@@ -280,10 +271,10 @@ func (w *walker) walkAndHashFiles(ctx context.Context, fchan, dchan chan protoco
return nil
case info.IsDir():
err = w.walkDir(ctx, relPath, info, dchan)
err = w.walkDir(ctx, path, info, dchan)
case info.IsRegular():
err = w.walkRegular(ctx, relPath, info, fchan)
err = w.walkRegular(ctx, path, info, fchan)
}
return err
@@ -375,7 +366,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
// walkSymlink returns nil or an error, if the error is of the nature that
// it should stop the entire walk.
func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan chan protocol.FileInfo) error {
func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan protocol.FileInfo) error {
// Symlinks are not supported on Windows. We ignore instead of returning
// an error.
if runtime.GOOS == "windows" {
@@ -387,9 +378,9 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
// checking that their existing blocks match with the blocks in
// the index.
target, err := w.Filesystem.ReadSymlink(absPath)
target, err := w.Filesystem.ReadSymlink(relPath)
if err != nil {
l.Debugln("readlink error:", absPath, err)
l.Debugln("readlink error:", relPath, err)
return nil
}
@@ -413,7 +404,7 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
SymlinkTarget: target,
}
l.Debugln("symlink changedb:", absPath, f)
l.Debugln("symlink changedb:", relPath, f)
select {
case dchan <- f:
@@ -426,55 +417,58 @@ func (w *walker) walkSymlink(ctx context.Context, absPath, relPath string, dchan
// normalizePath returns the normalized relative path (possibly after fixing
// it on disk), or skip is true.
func (w *walker) normalizePath(absPath, relPath string) (normPath string, skip bool) {
func (w *walker) normalizePath(path string) (normPath string, skip bool) {
if runtime.GOOS == "darwin" {
// Mac OS X file names should always be NFD normalized.
normPath = norm.NFD.String(relPath)
normPath = norm.NFD.String(path)
} else {
// Every other OS in the known universe uses NFC or just plain
// doesn't bother to define an encoding. In our case *we* do care,
// so we enforce NFC regardless.
normPath = norm.NFC.String(relPath)
normPath = norm.NFC.String(path)
}
if relPath != normPath {
if path != normPath {
// The file name was not normalized.
if !w.AutoNormalize {
// We're not authorized to do anything about it, so complain and skip.
l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", relPath)
l.Warnf("File name %q is not in the correct UTF8 normalization form; skipping.", path)
return "", true
}
// We will attempt to normalize it.
normalizedPath := filepath.Join(w.Dir, normPath)
if _, err := w.Filesystem.Lstat(normalizedPath); fs.IsNotExist(err) {
if _, err := w.Filesystem.Lstat(normPath); fs.IsNotExist(err) {
// Nothing exists with the normalized filename. Good.
if err = w.Filesystem.Rename(absPath, normalizedPath); err != nil {
l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, relPath, err)
if err = w.Filesystem.Rename(path, normPath); err != nil {
l.Infof(`Error normalizing UTF8 encoding of file "%s": %v`, path, err)
return "", true
}
l.Infof(`Normalized UTF8 encoding of file name "%s".`, relPath)
l.Infof(`Normalized UTF8 encoding of file name "%s".`, path)
} else {
// There is something already in the way at the normalized
// file name.
l.Infof(`File "%s" has UTF8 encoding conflict with another file; ignoring.`, relPath)
l.Infof(`File "%s" path has UTF8 encoding conflict with another file; ignoring.`, path)
return "", true
}
}
return normPath, false
return path, false
}
func (w *walker) checkDir() error {
if info, err := w.Filesystem.Lstat(w.Dir); err != nil {
info, err := w.Filesystem.Lstat(".")
if err != nil {
return err
} else if !info.IsDir() {
return errors.New(w.Dir + ": not a directory")
} else {
l.Debugln("checkDir", w.Dir, info)
}
if !info.IsDir() {
return errors.New(w.Filesystem.URI() + ": not a directory")
}
l.Debugln("checkDir", w.Filesystem.Type(), w.Filesystem.URI(), info)
return nil
}

68
lib/scanner/walk_test.go
View File

@@ -23,7 +23,6 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"golang.org/x/text/unicode/norm"
)
@@ -54,18 +53,18 @@ func init() {
}
func TestWalkSub(t *testing.T) {
ignores := ignore.New()
ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
}
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
Subs: []string{"dir2"},
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
Subs: []string{"dir2"},
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
var files []protocol.FileInfo
for f := range fchan {
@@ -90,7 +89,7 @@ func TestWalkSub(t *testing.T) {
}
func TestWalk(t *testing.T) {
ignores := ignore.New()
ignores := ignore.New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."))
err := ignores.Load("testdata/.stignore")
if err != nil {
t.Fatal(err)
@@ -98,10 +97,10 @@ func TestWalk(t *testing.T) {
t.Log(ignores)
fchan, err := Walk(context.TODO(), Config{
Dir: "testdata",
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
if err != nil {
@@ -122,9 +121,9 @@ func TestWalk(t *testing.T) {
func TestWalkError(t *testing.T) {
_, err := Walk(context.TODO(), Config{
Dir: "testdata-missing",
BlockSize: 128 * 1024,
Hashers: 2,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata-missing"),
BlockSize: 128 * 1024,
Hashers: 2,
})
if err == nil {
@@ -132,8 +131,8 @@ func TestWalkError(t *testing.T) {
}
_, err = Walk(context.TODO(), Config{
Dir: "testdata/bar",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata/bar"),
BlockSize: 128 * 1024,
})
if err == nil {
@@ -220,9 +219,11 @@ func TestNormalization(t *testing.T) {
numValid := len(tests) - numInvalid
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, ".")
for _, s1 := range tests {
// Create a directory for each of the interesting strings above
if err := osutil.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
if err := fs.MkdirAll(filepath.Join("testdata/normalization", s1), 0755); err != nil {
t.Fatal(err)
}
@@ -231,10 +232,10 @@ func TestNormalization(t *testing.T) {
// file names. Ensure that the file doesn't exist when it's
// created. This detects and fails if there's file name
// normalization stuff at the filesystem level.
if fd, err := os.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
if fd, err := fs.OpenFile(filepath.Join("testdata/normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
t.Fatal(err)
} else {
fd.WriteString("test")
fd.Write([]byte("test"))
fd.Close()
}
}
@@ -245,11 +246,11 @@ func TestNormalization(t *testing.T) {
// make sure it all gets done. In production, things will be correct
// eventually...
_, err := walkDir("testdata/normalization")
_, err := walkDir(fs, "testdata/normalization")
if err != nil {
t.Fatal(err)
}
tmp, err := walkDir("testdata/normalization")
tmp, err := walkDir(fs, "testdata/normalization")
if err != nil {
t.Fatal(err)
}
@@ -299,8 +300,8 @@ func TestWalkSymlinkUnix(t *testing.T) {
// Scan it
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
BlockSize: 128 * 1024,
})
if err != nil {
@@ -344,8 +345,8 @@ func TestWalkSymlinkWindows(t *testing.T) {
// Scan it
fchan, err := Walk(context.TODO(), Config{
Dir: "_symlinks",
BlockSize: 128 * 1024,
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"),
BlockSize: 128 * 1024,
})
if err != nil {
@@ -364,9 +365,10 @@ func TestWalkSymlinkWindows(t *testing.T) {
}
}
func walkDir(dir string) ([]protocol.FileInfo, error) {
func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) {
fchan, err := Walk(context.TODO(), Config{
Dir: dir,
Filesystem: fs,
Subs: []string{dir},
BlockSize: 128 * 1024,
AutoNormalize: true,
Hashers: 2,
@@ -435,7 +437,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.DefaultFilesystem, testdataName, protocol.BlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@@ -467,15 +469,17 @@ func TestStopWalk(t *testing.T) {
// many directories. It'll take a while to scan, giving us time to
// cancel it and make sure the scan stops.
fs := fs.NewWalkFilesystem(&infiniteFS{100, 100, 1e6})
// Use an errorFs as the backing fs for the rest of the interface
// The way we get it is a bit hacky tho.
errorFs := fs.NewFilesystem(fs.FilesystemType(-1), ".")
fs := fs.NewWalkFilesystem(&infiniteFS{errorFs, 100, 100, 1e6})
const numHashers = 4
ctx, cancel := context.WithCancel(context.Background())
fchan, err := Walk(ctx, Config{
Dir: "testdir",
Filesystem: fs,
BlockSize: 128 * 1024,
Hashers: numHashers,
Filesystem: fs,
ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
})