all: Add receive only folder type (#5027)

Adds a receive only folder type that does not send changes, and where the user can optionally revert local changes. Also changes some of the icons to make the three folder types distinguishable.
This commit is contained in:
Jakob Borg
2018-07-12 11:15:57 +03:00
committed by GitHub
parent 1a6c7587c2
commit f822b10550
29 changed files with 1136 additions and 144 deletions

View File

@@ -27,6 +27,7 @@ var errWatchNotStarted = errors.New("not started")
type folder struct {
stateTracker
config.FolderConfiguration
localFlags uint32
model *Model
shortID protocol.ShortID
@@ -175,6 +176,8 @@ func (f *folder) BringToFront(string) {}
func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) DelayScan(next time.Duration) {
f.Delay(next)
}
@@ -263,7 +266,7 @@ func (f *folder) getHealthError() error {
}
func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs, f.localFlags); err != nil {
// Potentially sets the error twice, once in the scanner just
// by doing a check, and once here, if the error returned is
// the same one as returned by CheckHealth, though

View File

@@ -0,0 +1,210 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"sort"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/versioner"
)
func init() {
folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder
}
/*
receiveOnlyFolder is a folder that does not propagate local changes outward.
It does this by the following general mechanism (not all of which is
implemted in this file):
- Local changes are scanned and versioned as usual, but get the
FlagLocalReceiveOnly bit set.
- When changes are sent to the cluster this bit gets converted to the
Invalid bit (like all other local flags, currently) and also the Version
gets set to the empty version. The reason for clearing the Version is to
ensure that other devices will not consider themselves out of date due to
our change.
- The database layer accounts sizes per flag bit, so we can know how many
files have been changed locally. We use this to trigger a "Revert" option
on the folder when the amount of locally changed data is nonzero.
- To revert we take the files which have changed and reset their version
counter down to zero. The next pull will replace our changed version with
the globally latest. As this is a user-initiated operation we do not cause
conflict copies when reverting.
- When pulling normally (i.e., not in the revert case) with local changes,
normal conflict resolution will apply. Conflict copies will be created,
but not propagated outwards (because receive only, right).
Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that
sets an extra bit on local changes and has a Revert method.
*/
type receiveOnlyFolder struct {
*sendReceiveFolder
}
func newReceiveOnlyFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
sr := newSendReceiveFolder(model, cfg, ver, fs).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr}
}
func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
// XXX: This *really* should be given to us in the constructor...
f.model.fmut.RLock()
ignores := f.model.folderIgnores[f.folderID]
f.model.fmut.RUnlock()
delQueue := &deleteQueue{
handler: f, // for the deleteFile and deleteDir methods
ignores: ignores,
}
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
// receive only mode.
return true
}
if len(fi.Version.Counters) == 1 && fi.Version.Counters[0].ID == f.shortID {
// We are the only device mentioned in the version vector so the
// file must originate here. A revert then means to delete it.
// We'll delete files directly, directories get queued and
// handled below.
handled, err := delQueue.handle(fi)
if err != nil {
l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
return true // continue
}
if !handled {
return true // continue
}
fi = protocol.FileInfo{
Name: fi.Name,
Type: fi.Type,
ModifiedS: fi.ModifiedS,
ModifiedNs: fi.ModifiedNs,
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{}, // if this file ever resurfaces anywhere we want our delete to be strictly older
}
} else {
// Revert means to throw away our local changes. We reset the
// version to the empty vector, which is strictly older than any
// other existing version. It is not in conflict with anything,
// either, so we will not create a conflict copy of our local
// changes.
fi.Version = protocol.Vector{}
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
batch = append(batch, fi)
batchSizeBytes += fi.ProtoSize()
if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes {
updateFn(batch)
batch = batch[:0]
batchSizeBytes = 0
}
return true
})
if len(batch) > 0 {
updateFn(batch)
}
batch = batch[:0]
batchSizeBytes = 0
// Handle any queued directories
deleted, err := delQueue.flush()
if err != nil {
l.Infoln("Revert:", err)
}
now := time.Now()
for _, dir := range deleted {
batch = append(batch, protocol.FileInfo{
Name: dir,
Type: protocol.FileInfoTypeDirectory,
ModifiedS: now.Unix(),
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{},
})
}
if len(batch) > 0 {
updateFn(batch)
}
// We will likely have changed our local index, but that won't trigger a
// pull by itself. Make sure we schedule one so that we start
// downloading files.
f.SchedulePull()
}
// deleteQueue handles deletes by delegating to a handler and queuing
// directories for last.
type deleteQueue struct {
handler interface {
deleteFile(file protocol.FileInfo) (dbUpdateJob, error)
deleteDir(dir string, ignores *ignore.Matcher, scanChan chan<- string) error
}
ignores *ignore.Matcher
dirs []string
}
func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
// Things that are ignored but not marked deletable are not processed.
ign := q.ignores.Match(fi.Name)
if ign.IsIgnored() && !ign.IsDeletable() {
return false, nil
}
// Directories are queued for later processing.
if fi.IsDirectory() {
q.dirs = append(q.dirs, fi.Name)
return false, nil
}
// Kill it.
_, err := q.handler.deleteFile(fi)
return true, err
}
func (q *deleteQueue) flush() ([]string, error) {
// Process directories from the leaves inward.
sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
var firstError error
var deleted []string
for _, dir := range q.dirs {
if err := q.handler.deleteDir(dir, q.ignores, nil); err == nil {
deleted = append(deleted, dir)
} else if err != nil && firstError == nil {
firstError = err
}
}
return deleted, firstError
}

View File

@@ -0,0 +1,261 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestRecvOnlyRevertDeletes(t *testing.T) {
// Make sure that we delete extraneous files and directories when we hit
// Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
os.MkdirAll("_recvonly/ignDir", 0755)
os.MkdirAll("_recvonly/unknownDir", 0755)
ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644)
knownFiles := setupKnownFiles(t, []byte("hello\n"))
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
// Start the folder. This will cause a scan, should discover the other stuff in the folder
m.StartFolder("ro")
m.ScanFolder("ro")
// We should now have two files and two directories.
size = m.GlobalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// Revert should delete the unknown stuff
m.Revert("ro")
// These should still exist
for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} {
_, err := os.Stat(p)
if err != nil {
t.Error("Unexpected error:", err)
}
}
// These should have been removed
for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} {
_, err := os.Stat(p)
if !os.IsNotExist(err) {
t.Error("Unexpected existing thing:", p)
}
}
// We should now have one file and directory again.
size = m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
}
}
func TestRecvOnlyRevertNeeds(t *testing.T) {
// Make sure that a new file gets picked up and considered latest, then
// gets considered old when we hit Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
oldData := []byte("hello\n")
knownFiles := setupKnownFiles(t, oldData)
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
// Start the folder. This will cause a scan.
m.StartFolder("ro")
m.ScanFolder("ro")
// Everything should be in sync.
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
// Update the file.
newData := []byte("totally different data\n")
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644); err != nil {
t.Fatal(err)
}
// Rescan.
if err := m.ScanFolder("ro"); err != nil {
t.Fatal(err)
}
// We now have a newer file than the rest of the cluster. Global state should reflect this.
size = m.GlobalSize("ro")
const sizeOfDir = 128
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Global: expected the new file to be reflected: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// We hit the Revert button. The file that was new should become old.
m.Revert("ro")
size = m.GlobalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected the global size to revert: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the local size to remain: %+v", size)
}
size = m.NeedSize("ro")
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
t.Fatalf("Local: expected to need the old file data: %+v", size)
}
}
func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo {
if err := os.MkdirAll("_recvonly/knownDir", 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644); err != nil {
t.Fatal(err)
}
t0 := time.Now().Add(-1 * time.Minute)
if err := os.Chtimes("_recvonly/knownDir/knownFile", t0, t0); err != nil {
t.Fatal(err)
}
fi, err := os.Stat("_recvonly/knownDir/knownFile")
if err != nil {
t.Fatal(err)
}
knownFiles := []protocol.FileInfo{
{
Name: "knownDir",
Type: protocol.FileInfoTypeDirectory,
Permissions: 0755,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
{
Name: "knownDir/knownFile",
Type: protocol.FileInfoTypeFile,
Permissions: 0644,
Size: fi.Size(),
ModifiedS: fi.ModTime().Unix(),
ModifiedNs: int32(fi.ModTime().UnixNano() % 1e9),
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
}
return knownFiles
}
func setupROFolder() *Model {
fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly")
fcfg.Type = config.FolderTypeReceiveOnly
fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
cfg := defaultCfg.Copy()
cfg.Folders = append(cfg.Folders, fcfg)
wrp := config.Wrap("/dev/null", cfg)
db := db.OpenMemory()
m := NewModel(wrp, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
m.ServeBackground()
m.AddFolder(fcfg)
return m
}

View File

@@ -76,7 +76,7 @@ func (f *sendOnlyFolder) pull() bool {
}
file := intf.(protocol.FileInfo)
if !file.IsEquivalent(curFile, f.IgnorePerms, false) {
if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) {
return true
}

View File

@@ -501,7 +501,11 @@ func (f *sendReceiveFolder) processDeletions(ignores *ignore.Matcher, fileDeleti
}
l.Debugln(f, "Deleting file", file.Name)
f.deleteFile(file, dbUpdateChan)
if update, err := f.deleteFile(file); err != nil {
f.newError("delete file", file.Name, err)
} else {
dbUpdateChan <- update
}
}
for i := range dirDeletions {
@@ -736,7 +740,7 @@ func (f *sendReceiveFolder) handleDeleteDir(file protocol.FileInfo, ignores *ign
}
// deleteFile attempts to delete the given file
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) {
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) (dbUpdateJob, error) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@@ -775,16 +779,18 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan
if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
return dbUpdateJob{file, dbUpdateDeleteFile}, nil
}
if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the file, and it's not a permission
// problem. Lets assume the error is in fact some variant of "file
// does not exist" (possibly expressed as some parent being a file and
// not a directory etc) and that the delete is handled.
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
} else {
f.newError("delete file", file.Name, err)
return dbUpdateJob{file, dbUpdateDeleteFile}, nil
}
return dbUpdateJob{}, err
}
// renameFile attempts to rename an existing file to a destination
@@ -1778,10 +1784,14 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC
} else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() {
hasIgnored = true
} else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
// Something appeared in the dir that we either are not
// aware of at all, that we think should be deleted or that
// is invalid, but not currently ignored -> schedule scan
scanChan <- fullDirFile
// Something appeared in the dir that we either are not aware of
// at all, that we think should be deleted or that is invalid,
// but not currently ignored -> schedule scan. The scanChan
// might be nil, in which case we trust the scanning to be
// handled later as a result of our error return.
if scanChan != nil {
scanChan <- fullDirFile
}
hasToBeScanned = true
} else {
// Dir contains file that is valid according to db and

View File

@@ -74,12 +74,12 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
}
}
func setUpModel(file protocol.FileInfo) *Model {
func setUpModel(files ...protocol.FileInfo) *Model {
db := db.OpenMemory()
model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
model.AddFolder(defaultFolderConfig)
// Update index
model.updateLocalsFromScanning("default", []protocol.FileInfo{file})
model.updateLocalsFromScanning("default", files)
return model
}

View File

@@ -57,6 +57,7 @@ const (
type service interface {
BringToFront(string)
Override(*db.FileSet, func([]protocol.FileInfo))
Revert(*db.FileSet, func([]protocol.FileInfo))
DelayScan(d time.Duration)
IgnoresUpdated() // ignore matcher was updated notification
SchedulePull() // something relevant changed, we should try a pull
@@ -690,6 +691,18 @@ func (m *Model) LocalSize(folder string) db.Counts {
return db.Counts{}
}
// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *Model) ReceiveOnlyChangedSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
return rf.ReceiveOnlyChangedSize()
}
return db.Counts{}
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
@@ -1747,6 +1760,12 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
// Mark the file as invalid if any of the local bad stuff flags are set.
f.RawInvalid = f.IsInvalid()
// If the file is marked LocalReceive (i.e., changed locally on a
// receive only folder) we do not want it to ever become the
// globally best version, invalid or not.
if f.IsReceiveOnlyChanged() {
f.Version = protocol.Vector{}
}
f.LocalFlags = 0 // never sent externally
if dropSymlinks && f.IsSymlink() {
@@ -1940,7 +1959,7 @@ func (m *Model) ScanFolderSubdirs(folder string, subs []string) error {
return runner.Scan(subs)
}
func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string) error {
func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string, localFlags uint32) error {
m.fmut.RLock()
if err := m.checkFolderRunningLocked(folder); err != nil {
m.fmut.RUnlock()
@@ -2010,6 +2029,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseLargeBlocks: folderCfg.UseLargeBlocks,
LocalFlags: localFlags,
})
if err := runner.CheckHealth(); err != nil {
@@ -2106,6 +2126,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ModifiedBy: m.id.Short(),
Deleted: true,
Version: f.Version.Update(m.shortID),
LocalFlags: localFlags,
}
// We do not want to override the global version
// with the deleted file. Keeping only our local
@@ -2289,6 +2310,24 @@ func (m *Model) Override(folder string) {
})
}
func (m *Model) Revert(folder string) {
// Grab the runner and the file set.
m.fmut.RLock()
fs, fsOK := m.folderFiles[folder]
runner, runnerOK := m.folderRunners[folder]
m.fmut.RUnlock()
if !fsOK || !runnerOK {
return
}
// Run the revert, taking updates as if they came from scanning.
runner.Revert(fs, func(files []protocol.FileInfo) {
m.updateLocalsFromScanning(folder, files)
})
}
// CurrentSequence returns the change version for the given folder.
// This is guaranteed to increment if the contents of the local folder has
// changed.