perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib: Handle metadata changes for send-only folders (fixes #4616, fixes #4627) (#4750)

Browse Source 
Unignored files are marked as conflicting while scanning, which is then resolved
in the subsequent pull. Automatically reconciles needed items on send-only
folders, if they do not actually differ except for internal metadata.
This commit is contained in:
Simon Frei
2018-02-25 09:39:00 +01:00
committed by Jakob Borg
parent 5822222c74
commit 158859a1e2
10 changed files with 425 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
lib/model/requests_test.go
View File

@@ -8,6 +8,7 @@ package model
import (
"bytes"
"errors"
"io/ioutil"
"os"
"path/filepath"
@@ -18,7 +19,9 @@ import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -26,9 +29,9 @@ func TestRequestSimple(t *testing.T) {
// Verify that the model performs a request and creates a file based on
// an incoming index update.
m, fc, tmpFolder := setupModelWithConnection()
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpFolder)
defer os.RemoveAll(tmpDir)
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
@@ -51,13 +54,8 @@ func TestRequestSimple(t *testing.T) {
<-done
// Verify the contents
bs, err := ioutil.ReadFile(filepath.Join(tmpFolder, "testfile"))
if err != nil {
if err := equalContents(filepath.Join(tmpDir, "testfile"), contents); err != nil {
t.Error("File did not sync correctly:", err)
return
}
if !bytes.Equal(bs, contents) {
t.Error("File did not sync correctly: incorrect data")
}
}
@@ -69,9 +67,9 @@ func TestSymlinkTraversalRead(t *testing.T) {
return
}
m, fc, tmpFolder := setupModelWithConnection()
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpFolder)
defer os.RemoveAll(tmpDir)
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
@@ -109,9 +107,9 @@ func TestSymlinkTraversalWrite(t *testing.T) {
return
}
m, fc, tmpFolder := setupModelWithConnection()
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpFolder)
defer os.RemoveAll(tmpDir)
// We listen for incoming index updates and trigger when we see one for
// the expected names.
@@ -169,9 +167,9 @@ func TestSymlinkTraversalWrite(t *testing.T) {
func TestRequestCreateTmpSymlink(t *testing.T) {
// Test that an update for a temporary file is invalidated
m, fc, tmpFolder := setupModelWithConnection()
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpFolder)
defer os.RemoveAll(tmpDir)
// We listen for incoming index updates and trigger when we see one for
// the expected test file.
@@ -211,12 +209,12 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
// Sets up a folder with trashcan versioning and tries to use a
// deleted symlink to escape
tmpFolder, err := ioutil.TempDir(".", "_request-")
tmpDir, err := ioutil.TempDir(".", "_request-")
if err != nil {
panic("Failed to create temporary testing dir")
}
cfg := defaultConfig.RawCopy()
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpFolder)
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpDir)
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
@@ -233,7 +231,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
m.StartFolder("default")
defer m.Stop()
defer os.RemoveAll(tmpFolder)
defer os.RemoveAll(tmpDir)
fc := addFakeConn(m, device2)
fc.folder = "default"
@@ -286,17 +284,161 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) {
}
}
func setupModelWithConnection() (*Model, *fakeConnection, string) {
tmpFolder, err := ioutil.TempDir(".", "_request-")
if err != nil {
panic("Failed to create temporary testing dir")
}
func TestPullInvalidIgnoredSO(t *testing.T) {
pullInvalidIgnored(t, config.FolderTypeSendOnly)
}
func TestPullInvalidIgnoredSR(t *testing.T) {
pullInvalidIgnored(t, config.FolderTypeSendReceive)
}
// This test checks that (un-)ignored/invalid/deleted files are treated as expected.
func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
t.Helper()
tmpDir := createTmpDir()
defer os.RemoveAll(tmpDir)
cfg := defaultConfig.RawCopy()
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpFolder)
cfg.Devices = append(cfg.Devices, config.NewDeviceConfiguration(device2, "device2"))
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpDir)
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
}
cfg.Folders[0].Type = ft
m, fc := setupModelWithConnectionManual(cfg)
defer m.Stop()
// Reach in and update the ignore matcher to one that always does
// reloads when asked to, instead of checking file mtimes. This is
// because we might be changing the files on disk often enough that the
// mtimes will be unreliable to determine change status.
m.fmut.Lock()
m.folderIgnores["default"] = ignore.New(cfg.Folders[0].Filesystem(), ignore.WithChangeDetector(newAlwaysChanged()))
m.fmut.Unlock()
if err := m.SetIgnores("default", []string{"*ignored*"}); err != nil {
panic(err)
}
contents := []byte("test file contents\n")
otherContents := []byte("other test file contents\n")
invIgn := "invalid:ignored"
invDel := "invalid:deleted"
ign := "ignoredNonExisting"
ignExisting := "ignoredExisting"
fc.addFile(invIgn, 0644, protocol.FileInfoTypeFile, contents)
fc.addFile(invDel, 0644, protocol.FileInfoTypeFile, contents)
fc.deleteFile(invDel)
fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents)
fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents)
if err := ioutil.WriteFile(filepath.Join(tmpDir, ignExisting), otherContents, 0644); err != nil {
panic(err)
}
done := make(chan struct{})
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
expected := map[string]struct{}{invIgn: {}, ign: {}, ignExisting: {}}
for _, f := range fs {
if _, ok := expected[f.Name]; !ok {
t.Fatalf("Unexpected file %v was added to index", f.Name)
}
if !f.Invalid {
t.Errorf("File %v wasn't marked as invalid", f.Name)
}
delete(expected, f.Name)
}
for name := range expected {
t.Errorf("File %v wasn't added to index", name)
}
done <- struct{}{}
}
fc.mut.Unlock()
sub := events.Default.Subscribe(events.FolderErrors)
defer events.Default.Unsubscribe(sub)
fc.sendIndexUpdate()
timeout := time.NewTimer(5 * time.Second)
select {
case ev := <-sub.C():
t.Fatalf("Errors while pulling: %v", ev)
case <-timeout.C:
t.Fatalf("timed out before index was received")
case <-done:
return
}
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
expected := map[string]struct{}{ign: {}, ignExisting: {}}
for _, f := range fs {
if _, ok := expected[f.Name]; !ok {
t.Fatalf("Unexpected file %v was updated in index", f.Name)
}
if f.Invalid {
t.Errorf("File %v is still marked as invalid", f.Name)
}
// The unignored files should only have a local version,
// to mark them as in conflict with any other existing versions.
ev := protocol.Vector{}.Update(device1.Short())
if v := f.Version; !v.Equal(ev) {
t.Errorf("File %v has version %v, expected %v", f.Name, v, ev)
}
if f.Name == ign {
if !f.Deleted {
t.Errorf("File %v was not marked as deleted", f.Name)
}
} else if f.Deleted {
t.Errorf("File %v is marked as deleted", f.Name)
}
delete(expected, f.Name)
}
for name := range expected {
t.Errorf("File %v wasn't updated in index", name)
}
done <- struct{}{}
}
// Make sure pulling doesn't interfere, as index updates are racy and
// thus we cannot distinguish between scan and pull results.
fc.requestFn = func(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
return nil, nil
}
fc.mut.Unlock()
if err := m.SetIgnores("default", []string{"*:ignored*"}); err != nil {
panic(err)
}
timeout = time.NewTimer(5 * time.Second)
select {
case <-timeout.C:
t.Fatalf("timed out before index was received")
case <-done:
return
}
}
func setupModelWithConnection() (*Model, *fakeConnection, string) {
tmpDir := createTmpDir()
cfg := defaultConfig.RawCopy()
cfg.Devices = append(cfg.Devices, config.NewDeviceConfiguration(device2, "device2"))
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpDir)
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{
{DeviceID: device1},
{DeviceID: device2},
}
m, fc := setupModelWithConnectionManual(cfg)
return m, fc, tmpDir
}
func setupModelWithConnectionManual(cfg config.Configuration) (*Model, *fakeConnection) {
w := config.Wrap("/tmp/cfg", cfg)
db := db.OpenMemory()
@@ -308,5 +450,24 @@ func setupModelWithConnection() (*Model, *fakeConnection, string) {
fc := addFakeConn(m, device2)
fc.folder = "default"
return m, fc, tmpFolder
m.ScanFolder("default")
return m, fc
}
func createTmpDir() string {
tmpDir, err := ioutil.TempDir(".", "_request-")
if err != nil {
panic("Failed to create temporary testing dir")
}
return tmpDir
}
func equalContents(path string, contents []byte) error {
if bs, err := ioutil.ReadFile(path); err != nil {
return err
} else if !bytes.Equal(bs, contents) {
return errors.New("incorrect data")
}
return nil
}