perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib: Add mtime window when comparing files (#5852)

Browse Source
This commit is contained in:
Simon Frei
2019-07-23 21:48:53 +02:00
committed by Jakob Borg
parent 35f40e9a58
commit 7a4c88d4e4
12 changed files with 129 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
lib/config/folderconfiguration.go
View File

@@ -10,6 +10,10 @@ import (
"errors"
"fmt"
"runtime"
"strings"
"time"
"github.com/shirou/gopsutil/disk"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
@@ -53,8 +57,10 @@ type FolderConfiguration struct {
WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
MarkerName string `xml:"markerName" json:"markerName"`
CopyOwnershipFromParent bool `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
RawModTimeWindowS int `xml:"modTimeWindowS" json:"modTimeWindowS"`
cachedFilesystem fs.Filesystem
cachedFilesystem fs.Filesystem
cachedModTimeWindow time.Duration
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@@ -111,6 +117,10 @@ func (f FolderConfiguration) Versioner() versioner.Versioner {
return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
}
func (f FolderConfiguration) ModTimeWindow() time.Duration {
return f.cachedModTimeWindow
}
func (f *FolderConfiguration) CreateMarker() error {
if err := f.CheckPath(); err != ErrMarkerMissing {
return err
@@ -233,6 +243,24 @@ func (f *FolderConfiguration) prepare() {
if f.MarkerName == "" {
f.MarkerName = DefaultMarkerName
}
switch {
case f.RawModTimeWindowS > 0:
f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
case runtime.GOOS == "android":
usage, err := disk.Usage(f.Filesystem().URI())
if err != nil {
l.Debugf("Error detecting FS at %v on android, setting mtime window to 2s: %v", f.Path, err)
f.cachedModTimeWindow = 2 * time.Second
break
}
if strings.Contains(strings.ToLower(usage.Fstype), "fat") {
l.Debugf("Detecting FS at %v on android, found %v, thus setting mtime window to 2s", f.Path, usage.Fstype)
f.cachedModTimeWindow = 2 * time.Second
break
}
l.Debugf("Detecting FS at %v on android, found %v, thus leaving mtime window at 0", f.Path, usage.Fstype)
}
}
// RequiresRestartOnly returns a copy with only the attributes that require

4
lib/db/db_test.go
View File

@@ -167,7 +167,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Unexpected additional file via sequence", f.FileName())
return true
}
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) {
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, 0, true, true, 0) {
found = true
} else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
@@ -192,7 +192,7 @@ func TestUpdate0to3(t *testing.T) {
}
f := fi.(protocol.FileInfo)
delete(need, f.Name)
if !f.IsEquivalentOptional(e, true, true, 0) {
if !f.IsEquivalentOptional(e, 0, true, true, 0) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
return true

16
lib/db/set_test.go
View File

@@ -905,7 +905,7 @@ func TestWithHaveSequence(t *testing.T) {
i := 2
s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
}
i++
@@ -1004,7 +1004,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Error("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(remote0Have[0]) {
} else if !need[0].IsEquivalent(remote0Have[0], 0) {
t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
}
@@ -1030,7 +1030,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, remoteDevice0); len(need) != 1 {
t.Error("Expected 1 need for remote 0, got", need)
} else if !need[0].IsEquivalent(localHave[0]) {
} else if !need[0].IsEquivalent(localHave[0], 0) {
t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
}
@@ -1066,7 +1066,7 @@ func TestIssue5007(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0]) {
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@@ -1101,7 +1101,7 @@ func TestNeedDeleted(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0]) {
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@@ -1243,7 +1243,7 @@ func TestNeedAfterUnignore(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need)
} else if !need[0].IsEquivalent(remote) {
} else if !need[0].IsEquivalent(remote, 0) {
t.Fatalf("Got %v, expected %v", need[0], remote)
}
}
@@ -1287,7 +1287,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
if len(need) != 1 {
t.Fatal("Locally missing file should be needed")
}
if !need[0].IsEquivalent(file) {
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
@@ -1302,7 +1302,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
if len(need) != 1 {
t.Fatal("Locally missing file should be needed regardless of invalid files")
}
if !need[0].IsEquivalent(file) {
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
}

3
lib/model/folder.go
View File

@@ -347,6 +347,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
ShortID: f.shortID,
ProgressTickIntervalS: f.ScanProgressIntervalS,
LocalFlags: f.localFlags,
ModTimeWindow: f.ModTimeWindow(),
})
batchFn := func(fs []protocol.FileInfo) error {
@@ -365,7 +366,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
switch gf, ok := f.fset.GetGlobal(fs[i].Name); {
case !ok:
continue
case gf.IsEquivalentOptional(fs[i], false, false, protocol.FlagLocalReceiveOnly):
case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly):
// What we have locally is equivalent to the global file.
fs[i].Version = fs[i].Version.Merge(gf.Version)
fallthrough

2
lib/model/folder_sendonly.go
View File

@@ -74,7 +74,7 @@ func (f *sendOnlyFolder) pull() bool {
}
file := intf.(protocol.FileInfo)
if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) {
if !file.IsEquivalentOptional(curFile, f.ModTimeWindow(), f.IgnorePerms, false, 0) {
return true
}

4
lib/model/folder_sendrecv.go
View File

@@ -953,7 +953,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
default:
var fi protocol.FileInfo
if fi, err = scanner.CreateFileInfo(stat, target.Name, f.fs); err == nil {
if !fi.IsEquivalentOptional(curTarget, f.IgnorePerms, true, protocol.LocalAllFlags) {
if !fi.IsEquivalentOptional(curTarget, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) {
// Target changed
scanChan <- target.Name
err = errModified
@@ -1919,7 +1919,7 @@ func (f *sendReceiveFolder) scanIfItemChanged(stat fs.FileInfo, item protocol.Fi
return errors.Wrap(err, "comparing item on disk to db")
}
if !statItem.IsEquivalentOptional(item, f.IgnorePerms, true, protocol.LocalAllFlags) {
if !statItem.IsEquivalentOptional(item, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) {
return errModified
}

52
lib/model/model_test.go
View File

@@ -3305,6 +3305,58 @@ func TestConnCloseOnRestart(t *testing.T) {
}
}
func TestModTimeWindow(t *testing.T) {
w, fcfg := tmpDefaultWrapper()
tfs := fcfg.Filesystem()
fcfg.RawModTimeWindowS = 2
w.SetFolder(fcfg)
m := setupModel(w)
defer cleanupModelAndRemoveDir(m, tfs.URI())
name := "foo"
fd, err := tfs.Create(name)
must(t, err)
stat, err := fd.Stat()
must(t, err)
modTime := stat.ModTime()
fd.Close()
m.ScanFolders()
v := protocol.Vector{}
v = v.Update(myID.Short())
fi, ok := m.CurrentFolderFile("default", name)
if !ok {
t.Fatal("File missing")
}
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
err = tfs.Chtimes(name, time.Now(), modTime.Add(time.Second))
must(t, err)
m.ScanFolders()
// No change due to window
fi, _ = m.CurrentFolderFile("default", name)
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
err = tfs.Chtimes(name, time.Now(), modTime.Add(2*time.Second))
must(t, err)
m.ScanFolders()
v = v.Update(myID.Short())
fi, _ = m.CurrentFolderFile("default", name)
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
}
func TestDevicePause(t *testing.T) {
sub := events.Default.Subscribe(events.DevicePaused)
defer events.Default.Unsubscribe(sub)

25
lib/protocol/bep_extensions.go
View File

@@ -158,12 +158,12 @@ func (f FileInfo) IsEmpty() bool {
return f.Version.Counters == nil
}
func (f FileInfo) IsEquivalent(other FileInfo) bool {
return f.isEquivalent(other, false, false, 0)
func (f FileInfo) IsEquivalent(other FileInfo, modTimeWindow time.Duration) bool {
return f.isEquivalent(other, modTimeWindow, false, false, 0)
}
func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags)
func (f FileInfo) IsEquivalentOptional(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, modTimeWindow, ignorePerms, ignoreBlocks, ignoreFlags)
}
// isEquivalent checks that the two file infos represent the same actual file content,
@@ -175,13 +175,13 @@ func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreB
// - invalid flag
// - permissions, unless they are ignored
// A file is not "equivalent", if it has different
// - modification time
// - modification time (difference bigger than modTimeWindow)
// - size
// - blocks, unless there are no blocks to compare (scanning)
// A symlink is not "equivalent", if it has different
// - target
// A directory does not have anything specific to check.
func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
func (f FileInfo) isEquivalent(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
if f.MustRescan() || other.MustRescan() {
// These are per definition not equivalent because they don't
// represent a valid state, even if both happen to have the
@@ -203,7 +203,7 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
switch f.Type {
case FileInfoTypeFile:
return f.Size == other.Size && f.ModTime().Equal(other.ModTime()) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks))
return f.Size == other.Size && ModTimeEqual(f.ModTime(), other.ModTime(), modTimeWindow) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks))
case FileInfoTypeSymlink:
return f.SymlinkTarget == other.SymlinkTarget
case FileInfoTypeDirectory:
@@ -213,6 +213,17 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
return false
}
func ModTimeEqual(a, b time.Time, modTimeWindow time.Duration) bool {
if a.Equal(b) {
return true
}
diff := a.Sub(b)
if diff < 0 {
diff *= -1
}
return diff < modTimeWindow
}
func PermsEqual(a, b uint32) bool {
switch runtime.GOOS {
case "windows":

4
lib/protocol/protocol_test.go
View File

@@ -770,10 +770,10 @@ func TestIsEquivalent(t *testing.T) {
continue
}
if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
if res := tc.a.isEquivalent(tc.b, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
if res := tc.b.isEquivalent(tc.a, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
}

8
lib/scanner/walk.go
View File

@@ -54,6 +54,8 @@ type Config struct {
ProgressTickIntervalS int
// Local flags to set on scanned files
LocalFlags uint32
// Modification time is to be considered unchanged if the difference is lower.
ModTimeWindow time.Duration
}
type CurrentFiler interface {
@@ -346,7 +348,7 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
f.RawBlockSize = int32(blockSize)
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {
@@ -379,7 +381,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
f.NoPermissions = w.IgnorePerms
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {
@@ -423,7 +425,7 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileIn
f = w.updateFileInfo(f, curFile)
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {