all: Add filesystem notification support

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3986

lib/fs/basicfs_test.go

@@ -7,12 +7,14 @@
 package fs
 
 import (
+	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 )
@@ -484,3 +486,23 @@ func TestRooted(t *testing.T) {
 		}
 	}
 }
+
+func TestWatchErrorLinuxInterpretation(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("testing of linux specific error codes")
+	}
+
+	var errTooManyFiles syscall.Errno = 24
+	var errNoSpace syscall.Errno = 28
+
+	if !reachedMaxUserWatches(errTooManyFiles) {
+		t.Errorf("Errno %v should be recognised to be about inotify limits.", errTooManyFiles)
+	}
+	if !reachedMaxUserWatches(errNoSpace) {
+		t.Errorf("Errno %v should be recognised to be about inotify limits.", errNoSpace)
+	}
+	err := errors.New("Another error")
+	if reachedMaxUserWatches(err) {
+		t.Errorf("This error does not concern inotify limits: %#v", err)
+	}
+}

lib/fs/basicfs_watch.go

@@ -0,0 +1,116 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !solaris,!darwin solaris,cgo darwin,cgo
+
+package fs
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+
+	"github.com/zillode/notify"
+)
+
+// Notify does not block on sending to channel, so the channel must be buffered.
+// The actual number is magic.
+// Not meant to be changed, but must be changeable for tests
+var backendBuffer = 500
+
+func (f *BasicFilesystem) Watch(name string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) {
+	absName, err := f.rooted(name)
+	if err != nil {
+		return nil, err
+	}
+
+	absShouldIgnore := func(absPath string) bool {
+		return ignore.ShouldIgnore(f.unrootedChecked(absPath))
+	}
+
+	outChan := make(chan Event)
+	backendChan := make(chan notify.EventInfo, backendBuffer)
+
+	eventMask := subEventMask
+	if !ignorePerms {
+		eventMask |= permEventMask
+	}
+
+	if err := notify.WatchWithFilter(filepath.Join(absName, "..."), backendChan, absShouldIgnore, eventMask); err != nil {
+		notify.Stop(backendChan)
+		if reachedMaxUserWatches(err) {
+			err = errors.New("failed to install inotify handler. Please increase inotify limits, see https://github.com/syncthing/syncthing-inotify#troubleshooting-for-folders-with-many-files-on-linux for more information")
+		}
+		return nil, err
+	}
+
+	go f.watchLoop(name, absName, backendChan, outChan, ignore, ctx)
+
+	return outChan, nil
+}
+
+func (f *BasicFilesystem) watchLoop(name string, absName string, backendChan chan notify.EventInfo, outChan chan<- Event, ignore Matcher, ctx context.Context) {
+	for {
+		// Detect channel overflow
+		if len(backendChan) == backendBuffer {
+		outer:
+			for {
+				select {
+				case <-backendChan:
+				default:
+					break outer
+				}
+			}
+			// When next scheduling a scan, do it on the entire folder as events have been lost.
+			outChan <- Event{Name: name, Type: NonRemove}
+			l.Debugln(f.Type(), f.URI(), "Watch: Event overflow, send \".\"")
+		}
+
+		select {
+		case ev := <-backendChan:
+			relPath := f.unrootedChecked(ev.Path())
+			if ignore.ShouldIgnore(relPath) {
+				l.Debugln(f.Type(), f.URI(), "Watch: Ignoring", relPath)
+				continue
+			}
+			evType := f.eventType(ev.Event())
+			select {
+			case outChan <- Event{Name: relPath, Type: evType}:
+				l.Debugln(f.Type(), f.URI(), "Watch: Sending", relPath, evType)
+			case <-ctx.Done():
+				notify.Stop(backendChan)
+				l.Debugln(f.Type(), f.URI(), "Watch: Stopped")
+				return
+			}
+		case <-ctx.Done():
+			notify.Stop(backendChan)
+			l.Debugln(f.Type(), f.URI(), "Watch: Stopped")
+			return
+		}
+	}
+}
+
+func (f *BasicFilesystem) eventType(notifyType notify.Event) EventType {
+	if notifyType&rmEventMask != 0 {
+		return Remove
+	}
+	return NonRemove
+}
+
+// unrootedChecked returns the path relative to the folder root (same as
+// unrooted). It panics if the given path is not a subpath and handles the
+// special case when the given path is the folder root without a trailing
+// pathseparator.
+func (f *BasicFilesystem) unrootedChecked(absPath string) string {
+	if absPath+string(PathSeparator) == f.root {
+		return "."
+	}
+	relPath := f.unrooted(absPath)
+	if relPath == absPath {
+		panic("bug: Notify backend is processing a change outside of the watched path: " + absPath)
+	}
+	return relPath
+}

lib/fs/basicfs_watch_errors_linux.go

@@ -0,0 +1,18 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build linux
+
+package fs
+
+import "syscall"
+
+func reachedMaxUserWatches(err error) bool {
+	if errno, ok := err.(syscall.Errno); ok {
+		return errno == 24 || errno == 28
+	}
+	return false
+}

lib/fs/basicfs_watch_errors_others.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !linux
+
+package fs
+
+func reachedMaxUserWatches(err error) bool {
+	return false
+}

lib/fs/basicfs_watch_eventtypes_fen.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build solaris,cgo
+
+package fs
+
+import "github.com/zillode/notify"
+
+const (
+	subEventMask  = notify.Create | notify.FileModified | notify.FileRenameFrom | notify.FileDelete | notify.FileRenameTo
+	permEventMask = notify.FileAttrib
+	rmEventMask   = notify.FileDelete | notify.FileRenameFrom
+)

lib/fs/basicfs_watch_eventtypes_inotify.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build linux
+
+package fs
+
+import "github.com/zillode/notify"
+
+const (
+	subEventMask  = notify.InCreate | notify.InMovedTo | notify.InDelete | notify.InDeleteSelf | notify.InModify | notify.InMovedFrom | notify.InMoveSelf
+	permEventMask = notify.InAttrib
+	rmEventMask   = notify.InDelete | notify.InDeleteSelf | notify.InMovedFrom | notify.InMoveSelf
+)

lib/fs/basicfs_watch_eventtypes_kqueue.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build dragonfly freebsd netbsd openbsd
+
+package fs
+
+import "github.com/zillode/notify"
+
+const (
+	subEventMask  = notify.NoteDelete | notify.NoteWrite | notify.NoteRename
+	permEventMask = notify.NoteAttrib
+	rmEventMask   = notify.NoteDelete | notify.NoteRename
+)

lib/fs/basicfs_watch_eventtypes_other.go

@@ -0,0 +1,21 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !linux,!windows,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris
+// +build !darwin darwin,cgo
+
+// Catch all platforms that are not specifically handled to use the generic
+// event types.
+
+package fs
+
+import "github.com/zillode/notify"
+
+const (
+	subEventMask  = notify.All
+	permEventMask = 0
+	rmEventMask   = notify.Remove | notify.Rename
+)

lib/fs/basicfs_watch_eventtypes_readdcw.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build windows
+
+package fs
+
+import "github.com/zillode/notify"
+
+const (
+	subEventMask  = notify.FileNotifyChangeFileName | notify.FileNotifyChangeDirName | notify.FileNotifyChangeSize | notify.FileNotifyChangeCreation
+	permEventMask = notify.FileNotifyChangeAttributes
+	rmEventMask   = notify.FileActionRemoved | notify.FileActionRenamedOldName
+)

lib/fs/basicfs_watch_test.go

@@ -0,0 +1,295 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !solaris,!darwin solaris,cgo darwin,cgo
+
+package fs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/zillode/notify"
+)
+
+func TestMain(m *testing.M) {
+	if err := os.RemoveAll(testDir); err != nil {
+		panic(err)
+	}
+
+	dir, err := filepath.Abs(".")
+	if err != nil {
+		panic("Cannot get absolute path to working dir")
+	}
+	dir, err = filepath.EvalSymlinks(dir)
+	if err != nil {
+		panic("Cannot get real path to working dir")
+	}
+	testDirAbs = filepath.Join(dir, testDir)
+	testFs = newBasicFilesystem(testDirAbs)
+	if l.ShouldDebug("filesystem") {
+		testFs = &logFilesystem{testFs}
+	}
+
+	backendBuffer = 10
+	defer func() {
+		backendBuffer = 500
+	}()
+	os.Exit(m.Run())
+}
+
+const (
+	testDir = "temporary_test_root"
+)
+
+var (
+	testDirAbs string
+	testFs     Filesystem
+)
+
+func TestWatchIgnore(t *testing.T) {
+	name := "ignore"
+
+	file := "file"
+	ignored := "ignored"
+
+	testCase := func() {
+		createTestFile(name, file)
+		createTestFile(name, ignored)
+	}
+
+	expectedEvents := []Event{
+		{file, NonRemove},
+	}
+
+	testScenario(t, name, testCase, expectedEvents, false, ignored)
+}
+
+func TestWatchRename(t *testing.T) {
+	name := "rename"
+
+	old := createTestFile(name, "oldfile")
+	new := "newfile"
+
+	testCase := func() {
+		renameTestFile(name, old, new)
+	}
+
+	destEvent := Event{new, Remove}
+	// Only on these platforms the removed file can be differentiated from
+	// the created file during renaming
+	if runtime.GOOS == "windows" || runtime.GOOS == "linux" || runtime.GOOS == "solaris" {
+		destEvent = Event{new, NonRemove}
+	}
+	expectedEvents := []Event{
+		{old, Remove},
+		destEvent,
+	}
+
+	testScenario(t, name, testCase, expectedEvents, false, "")
+}
+
+// TestWatchOutside checks that no changes from outside the folder make it in
+func TestWatchOutside(t *testing.T) {
+	outChan := make(chan Event)
+	backendChan := make(chan notify.EventInfo, backendBuffer)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// testFs is Filesystem, but we need BasicFilesystem here
+	fs := newBasicFilesystem(testDirAbs)
+
+	go func() {
+		defer func() {
+			if recover() == nil {
+				t.Fatalf("Watch did not panic on receiving event outside of folder")
+			}
+			cancel()
+		}()
+		fs.watchLoop(".", testDirAbs, backendChan, outChan, fakeMatcher{}, ctx)
+	}()
+
+	backendChan <- fakeEventInfo(filepath.Join(filepath.Dir(testDirAbs), "outside"))
+}
+
+func TestWatchSubpath(t *testing.T) {
+	outChan := make(chan Event)
+	backendChan := make(chan notify.EventInfo, backendBuffer)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// testFs is Filesystem, but we need BasicFilesystem here
+	fs := newBasicFilesystem(testDirAbs)
+
+	abs, _ := fs.rooted("sub")
+	go fs.watchLoop("sub", abs, backendChan, outChan, fakeMatcher{}, ctx)
+
+	backendChan <- fakeEventInfo(filepath.Join(abs, "file"))
+
+	timeout := time.NewTimer(2 * time.Second)
+	select {
+	case <-timeout.C:
+		t.Errorf("Timed out before receiving an event")
+		cancel()
+	case ev := <-outChan:
+		if ev.Name != filepath.Join("sub", "file") {
+			t.Errorf("While watching a subfolder, received an event with unexpected path %v", ev.Name)
+		}
+	}
+
+	cancel()
+}
+
+// TestWatchOverflow checks that an event at the root is sent when maxFiles is reached
+func TestWatchOverflow(t *testing.T) {
+	name := "overflow"
+
+	testCase := func() {
+		for i := 0; i < 5*backendBuffer; i++ {
+			createTestFile(name, "file"+strconv.Itoa(i))
+		}
+	}
+
+	expectedEvents := []Event{
+		{".", NonRemove},
+	}
+
+	testScenario(t, name, testCase, expectedEvents, true, "")
+}
+
+// path relative to folder root, also creates parent dirs if necessary
+func createTestFile(name string, file string) string {
+	joined := filepath.Join(name, file)
+	if err := testFs.MkdirAll(filepath.Dir(joined), 0755); err != nil {
+		panic(fmt.Sprintf("Failed to create parent directory for %s: %s", joined, err))
+	}
+	handle, err := testFs.Create(joined)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to create test file %s: %s", joined, err))
+	}
+	handle.Close()
+	return file
+}
+
+func renameTestFile(name string, old string, new string) {
+	old = filepath.Join(name, old)
+	new = filepath.Join(name, new)
+	if err := testFs.Rename(old, new); err != nil {
+		panic(fmt.Sprintf("Failed to rename %s to %s: %s", old, new, err))
+	}
+}
+
+func sleepMs(ms int) {
+	time.Sleep(time.Duration(ms) * time.Millisecond)
+}
+
+func testScenario(t *testing.T, name string, testCase func(), expectedEvents []Event, allowOthers bool, ignored string) {
+	if err := testFs.MkdirAll(name, 0755); err != nil {
+		panic(fmt.Sprintf("Failed to create directory %s: %s", name, err))
+	}
+
+	// Tests pick up the previously created files/dirs, probably because
+	// they get flushed to disk with a delay.
+	initDelayMs := 500
+	if runtime.GOOS == "darwin" {
+		initDelayMs = 900
+	}
+	sleepMs(initDelayMs)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	if ignored != "" {
+		ignored = filepath.Join(name, ignored)
+	}
+
+	eventChan, err := testFs.Watch(name, fakeMatcher{ignored}, ctx, false)
+	if err != nil {
+		panic(err)
+	}
+
+	go testWatchOutput(t, name, eventChan, expectedEvents, allowOthers, ctx, cancel)
+
+	timeout := time.NewTimer(2 * time.Second)
+
+	testCase()
+
+	select {
+	case <-timeout.C:
+		t.Errorf("Timed out before receiving all expected events")
+		cancel()
+	case <-ctx.Done():
+	}
+
+	if err := testFs.RemoveAll(name); err != nil {
+		panic(fmt.Sprintf("Failed to remove directory %s: %s", name, err))
+	}
+}
+
+func testWatchOutput(t *testing.T, name string, in <-chan Event, expectedEvents []Event, allowOthers bool, ctx context.Context, cancel context.CancelFunc) {
+	var expected = make(map[Event]struct{})
+	for _, ev := range expectedEvents {
+		ev.Name = filepath.Join(name, ev.Name)
+		expected[ev] = struct{}{}
+	}
+
+	var received Event
+	var last Event
+	for {
+		if len(expected) == 0 {
+			cancel()
+			return
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case received = <-in:
+		}
+
+		// apparently the backend sometimes sends repeat events
+		if last == received {
+			continue
+		}
+
+		if _, ok := expected[received]; !ok {
+			if allowOthers {
+				sleepMs(100) // To facilitate overflow
+				continue
+			}
+			t.Errorf("Received unexpected event %v expected one of %v", received, expected)
+			cancel()
+			return
+		}
+		delete(expected, received)
+		last = received
+	}
+}
+
+type fakeMatcher struct{ match string }
+
+func (fm fakeMatcher) ShouldIgnore(name string) bool {
+	return name == fm.match
+}
+
+type fakeEventInfo string
+
+func (e fakeEventInfo) Path() string {
+	return string(e)
+}
+
+func (e fakeEventInfo) Event() notify.Event {
+	return notify.Write
+}
+
+func (e fakeEventInfo) Sys() interface{} {
+	return nil
+}

lib/fs/basicfs_watch_unsupported.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build solaris,!cgo darwin,!cgo
+
+package fs
+
+import "context"
+
+func (f *BasicFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) {
+	return nil, ErrWatchNotSupported
+}

lib/fs/errorfs.go

@@ -6,7 +6,10 @@
 
 package fs
 
-import "time"
+import (
+	"context"
+	"time"
+)
 
 type errorFilesystem struct {
 	err    error
@@ -39,3 +42,6 @@ func (fs *errorFilesystem) Roots() ([]string, error)
 func (fs *errorFilesystem) Usage(name string) (Usage, error)                            { return Usage{}, fs.err }
 func (fs *errorFilesystem) Type() FilesystemType                                        { return fs.fsType }
 func (fs *errorFilesystem) URI() string                                                 { return fs.uri }
+func (fs *errorFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) {
+	return nil, fs.err
+}

lib/fs/filesystem.go

@@ -7,6 +7,7 @@
 package fs
 
 import (
+	"context"
 	"errors"
 	"io"
 	"os"
@@ -33,7 +34,8 @@ type Filesystem interface {
 	Rename(oldname, newname string) error
 	Stat(name string) (FileInfo, error)
 	SymlinksSupported() bool
-	Walk(root string, walkFn WalkFunc) error
+	Walk(name string, walkFn WalkFunc) error
+	Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error)
 	Hide(name string) error
 	Unhide(name string) error
 	Glob(pattern string) ([]string, error)
@@ -82,6 +84,42 @@ type Usage struct {
 	Total int64
 }
 
+type Matcher interface {
+	ShouldIgnore(name string) bool
+}
+
+type MatchResult interface {
+	IsIgnored() bool
+}
+
+type Event struct {
+	Name string
+	Type EventType
+}
+
+type EventType int
+
+const (
+	NonRemove EventType = 1 + iota
+	Remove
+	Mixed // Should probably not be necessary to be used in filesystem interface implementation
+)
+
+func (evType EventType) String() string {
+	switch {
+	case evType == NonRemove:
+		return "non-remove"
+	case evType == Remove:
+		return "remove"
+	case evType == Mixed:
+		return "mixed"
+	default:
+		panic("bug: Unknown event type")
+	}
+}
+
+var ErrWatchNotSupported = errors.New("watching is not supported")
+
 // Equivalents from os package.
 
 const ModePerm = FileMode(os.ModePerm)

lib/fs/logfs.go

@@ -7,6 +7,7 @@
 package fs
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"runtime"
@@ -127,6 +128,12 @@ func (fs *logFilesystem) Walk(root string, walkFn WalkFunc) error {
 	return err
 }
 
+func (fs *logFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, error) {
+	evChan, err := fs.Filesystem.Watch(path, ignore, ctx, ignorePerms)
+	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Watch", path, ignore, ignorePerms, err)
+	return evChan, err
+}
+
 func (fs *logFilesystem) Unhide(name string) error {
 	err := fs.Filesystem.Unhide(name)
 	l.Debugln(getCaller(), fs.Type(), fs.URI(), "Unhide", name, err)