perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing: UI for version restoration (fixes #2599) (#4602)

Browse Source 
cmd/syncthing: Add UI for version restoration (fixes #2599)
This commit is contained in:
Audrius Butkevicius
2018-01-01 14:39:23 +00:00
committed by Jakob Borg
parent c7f136c2b8
commit b0e2050cdb
33 changed files with 20045 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/config/folderconfiguration.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
var (
@@ -98,6 +99,18 @@ func (f FolderConfiguration) Filesystem() fs.Filesystem {
return f.cachedFilesystem
}
func (f FolderConfiguration) Versioner() versioner.Versioner {
if f.Versioning.Type == "" {
return nil
}
versionerFactory, ok := versioner.Factories[f.Versioning.Type]
if !ok {
l.Fatalf("Requested versioning type %q that does not exist", f.Versioning.Type)
}
return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
}
func (f *FolderConfiguration) CreateMarker() error {
if err := f.CheckPath(); err != errMarkerMissing {
return err

158
lib/model/model.go
View File

@@ -38,6 +38,16 @@ import (
"github.com/thejerf/suture"
)
var locationLocal *time.Location
func init() {
var err error
locationLocal, err = time.LoadLocation("Local")
if err != nil {
panic(err.Error())
}
}
// How many files to send in each Index/IndexUpdate message.
const (
maxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
@@ -232,21 +242,13 @@ func (m *Model) startFolderLocked(folder string) config.FolderType {
}
}
var ver versioner.Versioner
if len(cfg.Versioning.Type) > 0 {
versionerFactory, ok := versioner.Factories[cfg.Versioning.Type]
if !ok {
l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
}
ver = versionerFactory(folder, cfg.Filesystem(), cfg.Versioning.Params)
if service, ok := ver.(suture.Service); ok {
// The versioner implements the suture.Service interface, so
// expects to be run in the background in addition to being called
// when files are going to be archived.
token := m.Add(service)
m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
}
ver := cfg.Versioner()
if service, ok := ver.(suture.Service); ok {
// The versioner implements the suture.Service interface, so
// expects to be run in the background in addition to being called
// when files are going to be archived.
token := m.Add(service)
m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
}
ffs := fs.MtimeFS()
@@ -2376,6 +2378,132 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
return output
}
func (m *Model) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
fcfg, ok := m.cfg.Folder(folder)
if !ok {
return nil, errFolderMissing
}
files := make(map[string][]versioner.FileVersion)
filesystem := fcfg.Filesystem()
err := filesystem.Walk(".stversions", func(path string, f fs.FileInfo, err error) error {
// Skip root (which is ok to be a symlink)
if path == ".stversions" {
return nil
}
// Ignore symlinks
if f.IsSymlink() {
return fs.SkipDir
}
// No records for directories
if f.IsDir() {
return nil
}
// Strip .stversions prefix.
path = strings.TrimPrefix(path, ".stversions"+string(fs.PathSeparator))
name, tag := versioner.UntagFilename(path)
// Something invalid
if name == "" || tag == "" {
return nil
}
name = osutil.NormalizedFilename(name)
versionTime, err := time.ParseInLocation(versioner.TimeFormat, tag, locationLocal)
if err != nil {
return nil
}
files[name] = append(files[name], versioner.FileVersion{
VersionTime: versionTime.Truncate(time.Second),
ModTime: f.ModTime().Truncate(time.Second),
Size: f.Size(),
})
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func (m *Model) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
fcfg, ok := m.cfg.Folder(folder)
if !ok {
return nil, errFolderMissing
}
filesystem := fcfg.Filesystem()
ver := fcfg.Versioner()
restore := make(map[string]string)
errors := make(map[string]string)
// Validation
for file, version := range versions {
file = osutil.NativeFilename(file)
tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat)
versionedTaggedFilename := filepath.Join(".stversions", versioner.TagFilename(file, tag))
// Check that the thing we've been asked to restore is actually a file
// and that it exists.
if info, err := filesystem.Lstat(versionedTaggedFilename); err != nil {
errors[file] = err.Error()
continue
} else if !info.IsRegular() {
errors[file] = "not a file"
continue
}
// Check that the target location of where we are supposed to restore
// either does not exist, or is actually a file.
if info, err := filesystem.Lstat(file); err == nil && !info.IsRegular() {
errors[file] = "cannot replace a non-file"
continue
} else if err != nil && !fs.IsNotExist(err) {
errors[file] = err.Error()
continue
}
restore[file] = versionedTaggedFilename
}
// Execution
var err error
for target, source := range restore {
err = nil
if _, serr := filesystem.Lstat(target); serr == nil {
if ver != nil {
err = osutil.InWritableDir(ver.Archive, filesystem, target)
} else {
err = osutil.InWritableDir(filesystem.Remove, filesystem, target)
}
}
filesystem.MkdirAll(filepath.Dir(target), 0755)
if err == nil {
err = osutil.Copy(filesystem, source, target)
}
if err != nil {
errors[target] = err.Error()
continue
}
}
// Trigger scan
if !fcfg.FSWatcherEnabled {
m.ScanFolder(folder)
}
return errors, nil
}
func (m *Model) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []Availability {
// The slightly unusual locking sequence here is because we need to hold
// pmut for the duration (as the value returned from foldersFiles can

213
lib/model/model_test.go
View File

@@ -29,9 +29,11 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
srand "github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/versioner"
)
var device1, device2 protocol.DeviceID
@@ -2871,6 +2873,217 @@ func TestIssue4475(t *testing.T) {
}
}
func TestVersionRestore(t *testing.T) {
// We create a bunch of files which we restore
// In each file, we write the filename as the content
// We verify that the content matches at the expected filenames
// after the restore operation.
dir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
dbi := db.OpenMemory()
fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, dir)
fcfg.Versioning.Type = "simple"
fcfg.FSWatcherEnabled = false
filesystem := fcfg.Filesystem()
rawConfig := config.Configuration{
Folders: []config.FolderConfiguration{fcfg},
}
cfg := config.Wrap("/tmp/test", rawConfig)
m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", dbi, nil)
m.AddFolder(fcfg)
m.StartFolder("default")
m.ServeBackground()
defer m.Stop()
m.ScanFolder("default")
sentinel, err := time.ParseInLocation(versioner.TimeFormat, "20200101-010101", locationLocal)
if err != nil {
t.Fatal(err)
}
sentinelTag := sentinel.Format(versioner.TimeFormat)
for _, file := range []string{
// Versions directory
".stversions/file~20171210-040404.txt", // will be restored
".stversions/existing~20171210-040404", // exists, should expect to be archived.
".stversions/something~20171210-040404", // will become directory, hence error
".stversions/dir/file~20171210-040404.txt",
".stversions/dir/file~20171210-040405.txt",
".stversions/dir/file~20171210-040406.txt",
".stversions/very/very/deep/one~20171210-040406.txt", // lives deep down, no directory exists.
".stversions/dir/existing~20171210-040406.txt", // exists, should expect to be archived.
".stversions/dir/file.txt~20171210-040405", // incorrect tag format, ignored.
".stversions/dir/cat", // incorrect tag format, ignored.
// "file.txt" will be restored
"existing",
"something/file", // Becomes directory
"dir/file.txt",
"dir/existing.txt",
} {
if runtime.GOOS == "windows" {
file = filepath.FromSlash(file)
}
dir := filepath.Dir(file)
if err := filesystem.MkdirAll(dir, 0755); err != nil {
t.Fatal(err)
}
if fd, err := filesystem.Create(file); err != nil {
t.Fatal(err)
} else if _, err := fd.Write([]byte(file)); err != nil {
t.Fatal(err)
} else if err := fd.Close(); err != nil {
t.Fatal(err)
} else if err := filesystem.Chtimes(file, sentinel, sentinel); err != nil {
t.Fatal(err)
}
}
versions, err := m.GetFolderVersions("default")
if err != nil {
t.Fatal(err)
}
expectedVersions := map[string]int{
"file.txt": 1,
"existing": 1,
"something": 1,
"dir/file.txt": 3,
"dir/existing.txt": 1,
"very/very/deep/one.txt": 1,
}
for name, vers := range versions {
cnt, ok := expectedVersions[name]
if !ok {
t.Errorf("unexpected %s", name)
}
if len(vers) != cnt {
t.Errorf("%s: %s != %s", name, cnt, len(vers))
}
// Delete, so we can check if we didn't hit something we expect afterwards.
delete(expectedVersions, name)
}
for name := range expectedVersions {
t.Errorf("not found expected %s", name)
}
// Restoring non existing folder fails.
_, err = m.RestoreFolderVersions("does not exist", nil)
if err == nil {
t.Errorf("expected an error")
}
makeTime := func(s string) time.Time {
tm, err := time.ParseInLocation(versioner.TimeFormat, s, locationLocal)
if err != nil {
t.Error(err)
}
return tm.Truncate(time.Second)
}
restore := map[string]time.Time{
"file.txt": makeTime("20171210-040404"),
"existing": makeTime("20171210-040404"),
"something": makeTime("20171210-040404"),
"dir/file.txt": makeTime("20171210-040406"),
"dir/existing.txt": makeTime("20171210-040406"),
"very/very/deep/one.txt": makeTime("20171210-040406"),
}
ferr, err := m.RestoreFolderVersions("default", restore)
if err != nil {
t.Fatal(err)
}
if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot replace a non-file" {
t.Fatalf("incorrect error or count: %d %s", len(ferr), ferr)
}
// Failed items are not expected to be restored.
// Remove them from expectations
for name := range ferr {
delete(restore, name)
}
// Check that content of files matches to the version they've been restored.
for file, version := range restore {
if runtime.GOOS == "windows" {
file = filepath.FromSlash(file)
}
tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat)
taggedName := filepath.Join(".stversions", versioner.TagFilename(file, tag))
fd, err := filesystem.Open(file)
if err != nil {
t.Error(err)
}
defer fd.Close()
content, err := ioutil.ReadAll(fd)
if err != nil {
t.Error(err)
}
if !bytes.Equal(content, []byte(taggedName)) {
t.Errorf("%s: %s != %s", file, string(content), taggedName)
}
}
// Simple versioner uses modtime for timestamp generation, so we can check
// if existing stuff was correctly archived as we restored.
expectArchived := map[string]struct{}{
"existing": {},
"dir/file.txt": {},
"dir/existing.txt": {},
}
// Even if they are at the archived path, content should have the non
// archived name.
for file := range expectArchived {
if runtime.GOOS == "windows" {
file = filepath.FromSlash(file)
}
taggedName := versioner.TagFilename(file, sentinelTag)
taggedArchivedName := filepath.Join(".stversions", taggedName)
fd, err := filesystem.Open(taggedArchivedName)
if err != nil {
t.Fatal(err)
}
defer fd.Close()
content, err := ioutil.ReadAll(fd)
if err != nil {
t.Error(err)
}
if !bytes.Equal(content, []byte(file)) {
t.Errorf("%s: %s != %s", file, string(content), file)
}
}
// Check for other unexpected things that are tagged.
filesystem.Walk(".", func(path string, f fs.FileInfo, err error) error {
if !f.IsRegular() {
return nil
}
if strings.Contains(path, sentinelTag) {
path = osutil.NormalizedFilename(path)
name, _ := versioner.UntagFilename(path)
name = strings.TrimPrefix(name, ".stversions/")
if _, ok := expectArchived[name]; !ok {
t.Errorf("unexpected file with sentinel tag: %s", name)
}
}
return nil
})
}
func TestPausedFolders(t *testing.T) {
// Create a separate wrapper not to pollute other tests.
cfg := defaultConfig.RawCopy()

2
lib/model/rwfolder.go
View File

@@ -1451,7 +1451,7 @@ func (f *sendReceiveFolder) performFinish(ignores *ignore.Matcher, state *shared
// file before we replace it. Archiving a non-existent file is not
// an error.
if err = f.versioner.Archive(state.file.Name); err != nil {
if err = osutil.InWritableDir(f.versioner.Archive, f.fs, state.file.Name); err != nil {
return err
}
}

4
lib/versioner/simple.go
View File

@@ -77,7 +77,7 @@ func (v Simple) Archive(filePath string) error {
return err
}
ver := taggedFilename(file, info.ModTime().Format(TimeFormat))
ver := TagFilename(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(dir, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(v.fs, filePath, dst)
@@ -86,7 +86,7 @@ func (v Simple) Archive(filePath string) error {
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
pattern := filepath.Join(dir, TagFilename(file, TimeGlob))
newVersions, err := v.fs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)

4
lib/versioner/simple_test.go
View File

@@ -34,14 +34,14 @@ func TestTaggedFilename(t *testing.T) {
for _, tc := range cases {
if tc[0] != "" {
// Test tagger
tf := taggedFilename(tc[0], tc[1])
tf := TagFilename(tc[0], tc[1])
if tf != tc[2] {
t.Errorf("%s != %s", tf, tc[2])
}
}
// Test parser
tag := filenameTag(tc[2])
tag := ExtractTag(tc[2])
if tag != tc[1] {
t.Errorf("%s != %s", tag, tc[1])
}

17
lib/versioner/staggered.go
View File

@@ -124,12 +124,13 @@ func (v *Staggered) clean() {
}
// Regular file, or possibly a symlink.
ext := filepath.Ext(path)
versionTag := filenameTag(path)
withoutExt := path[:len(path)-len(ext)-len(versionTag)-1]
name := withoutExt + ext
dirTracker.addFile(path)
name, _ := UntagFilename(path)
if name == "" {
return nil
}
versionsPerFile[name] = append(versionsPerFile[name], path)
return nil
@@ -173,7 +174,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
var remove []string
for _, file := range versions {
loc, _ := time.LoadLocation("Local")
versionTime, err := time.ParseInLocation(TimeFormat, filenameTag(file), loc)
versionTime, err := time.ParseInLocation(TimeFormat, ExtractTag(file), loc)
if err != nil {
l.Debugf("Versioner: file name %q is invalid: %v", file, err)
continue
@@ -258,7 +259,7 @@ func (v *Staggered) Archive(filePath string) error {
return err
}
ver := taggedFilename(file, time.Now().Format(TimeFormat))
ver := TagFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
@@ -273,7 +274,7 @@ func (v *Staggered) Archive(filePath string) error {
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob))
pattern := filepath.Join(inFolderPath, TagFilename(file, TimeGlob))
newVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)

19
lib/versioner/util.go
View File

@@ -9,10 +9,11 @@ package versioner
import (
"path/filepath"
"regexp"
"strings"
)
// Inserts ~tag just before the extension of the filename.
func taggedFilename(name, tag string) string {
func TagFilename(name, tag string) string {
dir, file := filepath.Dir(name), filepath.Base(name)
ext := filepath.Ext(file)
withoutExt := file[:len(file)-len(ext)]
@@ -22,7 +23,7 @@ func taggedFilename(name, tag string) string {
var tagExp = regexp.MustCompile(`.*~([^~.]+)(?:\.[^.]+)?$`)
// Returns the tag from a filename, whether at the end or middle.
func filenameTag(path string) string {
func ExtractTag(path string) string {
match := tagExp.FindStringSubmatch(path)
// match is []string{"whole match", "submatch"} when successful
@@ -31,3 +32,17 @@ func filenameTag(path string) string {
}
return match[1]
}
func UntagFilename(path string) (string, string) {
ext := filepath.Ext(path)
versionTag := ExtractTag(path)
// Files tagged with old style tags cannot be untagged.
if versionTag == "" || strings.HasSuffix(ext, versionTag) {
return "", ""
}
withoutExt := path[:len(path)-len(ext)-len(versionTag)-1]
name := withoutExt + ext
return name, versionTag
}

12
lib/versioner/versioner.go
View File

@@ -8,12 +8,22 @@
// simple default versioning scheme.
package versioner
import "github.com/syncthing/syncthing/lib/fs"
import (
"time"
"github.com/syncthing/syncthing/lib/fs"
)
type Versioner interface {
Archive(filePath string) error
}
type FileVersion struct {
VersionTime time.Time `json:"versionTime"`
ModTime time.Time `json:"modTime"`
Size int64 `json:"size"`
}
var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
const (