perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/versioner: Restore for all versioners, cross-device support (#5514)

Browse Source 
* lib/versioner: Restore for all versioners, cross-device support

Fixes #4631
Fixes #4586
Fixes #1634
Fixes #5338
Fixes #5419
This commit is contained in:
Audrius Butkevicius
2019-04-28 23:30:16 +01:00
committed by GitHub
parent 2984d40641
commit 0ca1f26ff8
14 changed files with 636 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lib/versioner/external.go
View File

@@ -12,6 +12,7 @@ import (
"os/exec"
"runtime"
"strings"
"time"
"github.com/syncthing/syncthing/lib/fs"
@@ -103,3 +104,11 @@ func (v External) Archive(filePath string) error {
}
return errors.New("Versioner: file was not removed by external script")
}
func (v External) GetVersions() (map[string][]FileVersion, error) {
return nil, ErrRestorationNotSupported
}
func (v External) Restore(filePath string, versionTime time.Time) error {
return ErrRestorationNotSupported
}

68
lib/versioner/simple.go
View File

@@ -9,9 +9,9 @@ package versioner
import (
"path/filepath"
"strconv"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
)
@@ -21,19 +21,21 @@ func init() {
}
type Simple struct {
keep int
fs fs.Filesystem
keep int
folderFs fs.Filesystem
versionsFs fs.Filesystem
}
func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
func NewSimple(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
keep, err := strconv.Atoi(params["keep"])
if err != nil {
keep = 5 // A reasonable default
}
s := Simple{
keep: keep,
fs: fs,
keep: keep,
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
}
l.Debugf("instantiated %#v", s)
@@ -43,51 +45,17 @@ func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Vers
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v Simple) Archive(filePath string) error {
info, err := v.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename)
if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := ".stversions"
_, err = v.fs.Stat(versionsDir)
if err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir .stversions")
v.fs.Mkdir(versionsDir, 0755)
v.fs.Hide(versionsDir)
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
dir := filepath.Join(versionsDir, inFolderPath)
err = v.fs.MkdirAll(dir, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := TagFilename(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(dir, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(v.fs, filePath, dst)
if err != nil {
return err
}
dir := filepath.Dir(filePath)
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, TagFilename(file, TimeGlob))
newVersions, err := v.fs.Glob(pattern)
newVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@@ -95,7 +63,7 @@ func (v Simple) Archive(filePath string) error {
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := v.fs.Glob(pattern)
oldVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@@ -108,7 +76,7 @@ func (v Simple) Archive(filePath string) error {
if len(versions) > v.keep {
for _, toRemove := range versions[:len(versions)-v.keep] {
l.Debugln("cleaning out", toRemove)
err = v.fs.Remove(toRemove)
err = v.versionsFs.Remove(toRemove)
if err != nil {
l.Warnln("removing old version:", err)
}
@@ -117,3 +85,11 @@ func (v Simple) Archive(filePath string) error {
return nil
}
func (v Simple) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(v.versionsFs)
}
func (v Simple) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}

65
lib/versioner/staggered.go
View File

@@ -7,7 +7,6 @@
package versioner
import (
"os"
"path/filepath"
"strconv"
"time"
@@ -48,16 +47,9 @@ func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]str
cleanInterval = 3600 // Default: clean once per hour
}
// Use custom path if set, otherwise .stversions in folderPath
var versionsFs fs.Filesystem
if params["versionsPath"] == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if filepath.IsAbs(params["versionsPath"]) {
versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
} else {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
}
l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
// Backwards compatibility
params["fsPath"] = params["versionsPath"]
versionsFs := fsFromParams(folderFs, params)
s := &Staggered{
cleanInterval: cleanInterval,
@@ -225,53 +217,12 @@ func (v *Staggered) Archive(filePath string) error {
v.mutex.Lock()
defer v.mutex.Unlock()
info, err := v.folderFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
if err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
if _, err := v.versionsFs.Stat("."); err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsFs)
v.versionsFs.MkdirAll(".", 0755)
v.versionsFs.Hide(".")
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
if err != nil {
return err
}
err = v.versionsFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := TagFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
/// TODO: Fix this when we have an alternative filesystem implementation
if v.versionsFs.Type() != fs.FilesystemTypeBasic {
panic("bug: staggered versioner used with unsupported filesystem")
}
err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(inFolderPath, TagFilename(file, TimeGlob))
@@ -295,3 +246,11 @@ func (v *Staggered) Archive(filePath string) error {
return nil
}
func (v *Staggered) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(v.versionsFs)
}
func (v *Staggered) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}

96
lib/versioner/trashcan.go
View File

@@ -8,12 +8,10 @@ package versioner
import (
"fmt"
"path/filepath"
"strconv"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func init() {
@@ -22,17 +20,19 @@ func init() {
}
type Trashcan struct {
fs fs.Filesystem
folderFs fs.Filesystem
versionsFs fs.Filesystem
cleanoutDays int
stop chan struct{}
}
func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
func NewTrashcan(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
// On error we default to 0, "do not clean out the trash can"
s := &Trashcan{
fs: fs,
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
cleanoutDays: cleanoutDays,
stop: make(chan struct{}),
}
@@ -44,49 +44,9 @@ func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Ve
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (t *Trashcan) Archive(filePath string) error {
info, err := t.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := ".stversions"
if _, err := t.fs.Stat(versionsDir); err != nil {
if !fs.IsNotExist(err) {
return err
}
l.Debugln("creating versions dir", versionsDir)
if err := t.fs.MkdirAll(versionsDir, 0777); err != nil {
return err
}
t.fs.Hide(versionsDir)
}
l.Debugln("archiving", filePath)
archivedPath := filepath.Join(versionsDir, filePath)
if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) {
return err
}
l.Debugln("moving to", archivedPath)
if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil {
return err
}
// Set the mtime to the time the file was deleted. This is used by the
// cleanout routine. If this fails things won't work optimally but there's
// not much we can do about it so we ignore the error.
t.fs.Chtimes(archivedPath, time.Now(), time.Now())
return nil
return archiveFile(t.folderFs, t.versionsFs, filePath, func(name, tag string) string {
return name
})
}
func (t *Trashcan) Serve() {
@@ -124,8 +84,7 @@ func (t *Trashcan) String() string {
}
func (t *Trashcan) cleanoutArchive() error {
versionsDir := ".stversions"
if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) {
if _, err := t.versionsFs.Lstat("."); fs.IsNotExist(err) {
return nil
}
@@ -144,20 +103,45 @@ func (t *Trashcan) cleanoutArchive() error {
if info.ModTime().Before(cutoff) {
// The file is too old; remove it.
t.fs.Remove(path)
err = t.versionsFs.Remove(path)
} else {
// Keep this file, and remember it so we don't unnecessarily try
// to remove this directory.
dirTracker.addFile(path)
}
return nil
}
if err := t.fs.Walk(versionsDir, walkFn); err != nil {
return err
}
dirTracker.deleteEmptyDirs(t.fs)
if err := t.versionsFs.Walk(".", walkFn); err != nil {
return err
}
dirTracker.deleteEmptyDirs(t.versionsFs)
return nil
}
func (t *Trashcan) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(t.versionsFs)
}
func (t *Trashcan) Restore(filepath string, versionTime time.Time) error {
// If we have an untagged file A and want to restore it on top of existing file A, we can't first archive the
// existing A as we'd overwrite the old A version, therefore when we archive existing file, we archive it with a
// tag but when the restoration is finished, we rename it (untag it). This is only important if when restoring A,
// there already exists a file at the same location
taggedName := ""
tagger := func(name, tag string) string {
// We can't use TagFilename here, as restoreFii would discover that as a valid version and restore that instead.
taggedName = fs.TempName(name)
return taggedName
}
err := restoreFile(t.versionsFs, t.folderFs, filepath, versionTime, tagger)
if taggedName == "" {
return err
}
return t.versionsFs.Rename(taggedName, filepath)
}

84
lib/versioner/trashcan_test.go
View File

@@ -75,3 +75,87 @@ func TestTrashcanCleanout(t *testing.T) {
t.Error("empty directory should have been removed")
}
}
func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) {
// This tests that trashcan versioner restoration correctly archives existing file, because trashcan versioner
// files are untagged, archiving existing file to replace with a restored version technically should collide in
// in names.
tmpDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
tmpDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
folderFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir1)
versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2)
writeFile(t, folderFs, "file", "A")
versioner := NewTrashcan("", folderFs, map[string]string{
"fsType": "basic",
"fsPath": tmpDir2,
})
if err := versioner.Archive("file"); err != nil {
t.Fatal(err)
}
if _, err := folderFs.Stat("file"); !fs.IsNotExist(err) {
t.Fatal(err)
}
versionInfo, err := versionsFs.Stat("file")
if err != nil {
t.Fatal(err)
}
if content := readFile(t, versionsFs, "file"); content != "A" {
t.Errorf("expected A got %s", content)
}
writeFile(t, folderFs, "file", "B")
if err := versioner.Restore("file", versionInfo.ModTime().Truncate(time.Second)); err != nil {
t.Fatal(err)
}
if content := readFile(t, folderFs, "file"); content != "A" {
t.Errorf("expected A got %s", content)
}
if content := readFile(t, versionsFs, "file"); content != "B" {
t.Errorf("expected B got %s", content)
}
}
func readFile(t *testing.T, filesystem fs.Filesystem, name string) string {
fd, err := filesystem.Open(name)
if err != nil {
t.Fatal(err)
}
defer fd.Close()
buf, err := ioutil.ReadAll(fd)
if err != nil {
t.Fatal(err)
}
return string(buf)
}
func writeFile(t *testing.T, filesystem fs.Filesystem, name, content string) {
fd, err := filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate, 0777)
if err != nil {
t.Fatal(err)
}
defer fd.Close()
if err := fd.Truncate(int64(len(content))); err != nil {
t.Fatal(err)
}
if n, err := fd.Write([]byte(content)); err != nil || n != len(content) {
t.Fatal(n, len(content), err)
}
}

225
lib/versioner/util.go
View File

@@ -7,11 +7,30 @@
package versioner
import (
"fmt"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
var locationLocal *time.Location
var errDirectory = fmt.Errorf("cannot restore on top of a directory")
var errNotFound = fmt.Errorf("version not found")
var errFileAlreadyExists = fmt.Errorf("file already exists")
func init() {
var err error
locationLocal, err = time.LoadLocation("Local")
if err != nil {
panic(err.Error())
}
}
// Inserts ~tag just before the extension of the filename.
func TagFilename(name, tag string) string {
dir, file := filepath.Dir(name), filepath.Base(name)
@@ -38,11 +57,215 @@ func UntagFilename(path string) (string, string) {
versionTag := ExtractTag(path)
// Files tagged with old style tags cannot be untagged.
if versionTag == "" || strings.HasSuffix(ext, versionTag) {
if versionTag == "" {
return "", ""
}
// Old style tag
if strings.HasSuffix(ext, versionTag) {
return strings.TrimSuffix(path, "~"+versionTag), versionTag
}
withoutExt := path[:len(path)-len(ext)-len(versionTag)-1]
name := withoutExt + ext
return name, versionTag
}
func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error) {
files := make(map[string][]FileVersion)
err := fileSystem.Walk(".", func(path string, f fs.FileInfo, err error) error {
// Skip root (which is ok to be a symlink)
if path == "." {
return nil
}
// Skip walking if we cannot walk...
if err != nil {
return err
}
// Ignore symlinks
if f.IsSymlink() {
return fs.SkipDir
}
// No records for directories
if f.IsDir() {
return nil
}
path = osutil.NormalizedFilename(path)
name, tag := UntagFilename(path)
// Something invalid, assume it's an untagged file
if name == "" || tag == "" {
versionTime := f.ModTime().Truncate(time.Second)
files[path] = append(files[path], FileVersion{
VersionTime: versionTime,
ModTime: versionTime,
Size: f.Size(),
})
return nil
}
versionTime, err := time.ParseInLocation(TimeFormat, tag, locationLocal)
if err != nil {
// Can't parse it, welp, continue
return nil
}
if err == nil {
files[name] = append(files[name], FileVersion{
VersionTime: versionTime.Truncate(time.Second),
ModTime: f.ModTime().Truncate(time.Second),
Size: f.Size(),
})
}
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
type fileTagger func(string, string) string
func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger) error {
filePath = osutil.NativeFilename(filePath)
info, err := srcFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
_, err = dstFs.Stat(".")
if err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir")
err := dstFs.Mkdir(".", 0755)
if err != nil {
return err
}
_ = dstFs.Hide(".")
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
err = dstFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := tagger(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
err = osutil.RenameOrCopy(srcFs, dstFs, filePath, dst)
// Set the mtime to the time the file was deleted. This can be used by the
// cleanout routine. If this fails things won't work optimally but there's
// not much we can do about it so we ignore the error.
_ = dstFs.Chtimes(dst, time.Now(), time.Now())
return err
}
func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time, tagger fileTagger) error {
// If the something already exists where we are restoring to, archive existing file for versioning
// remove if it's a symlink, or fail if it's a directory
if info, err := dst.Lstat(filePath); err == nil {
switch {
case info.IsDir():
return errDirectory
case info.IsSymlink():
// Remove existing symlinks (as we don't want to archive them)
if err := dst.Remove(filePath); err != nil {
return errors.Wrap(err, "removing existing symlink")
}
case info.IsRegular():
if err := archiveFile(dst, src, filePath, tagger); err != nil {
return errors.Wrap(err, "archiving existing file")
}
default:
panic("bug: unknown item type")
}
} else if !fs.IsNotExist(err) {
return err
}
filePath = osutil.NativeFilename(filePath)
tag := versionTime.In(locationLocal).Truncate(time.Second).Format(TimeFormat)
taggedFilename := TagFilename(filePath, tag)
oldTaggedFilename := filePath + tag
untaggedFileName := filePath
// Check that the thing we've been asked to restore is actually a file
// and that it exists.
sourceFile := ""
for _, candidate := range []string{taggedFilename, oldTaggedFilename, untaggedFileName} {
if info, err := src.Lstat(candidate); fs.IsNotExist(err) || !info.IsRegular() {
continue
} else if err != nil {
// All other errors are fatal
return err
} else if candidate == untaggedFileName && !info.ModTime().Truncate(time.Second).Equal(versionTime) {
// No error, and untagged file, but mtime does not match, skip
continue
}
sourceFile = candidate
break
}
if sourceFile == "" {
return errNotFound
}
// Check that the target location of where we are supposed to restore does not exist.
// This should have been taken care of by the first few lines of this function.
if _, err := dst.Lstat(filePath); err == nil {
return errFileAlreadyExists
} else if !fs.IsNotExist(err) {
return err
}
_ = dst.MkdirAll(filepath.Dir(filePath), 0755)
return osutil.RenameOrCopy(src, dst, sourceFile, filePath)
}
func fsFromParams(folderFs fs.Filesystem, params map[string]string) (versionsFs fs.Filesystem) {
if params["fsType"] == "" && params["fsPath"] == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if params["fsType"] == "" {
uri := params["fsPath"]
// We only know how to deal with relative folders for basic filesystems, as that's the only one we know
// how to check if it's absolute or relative.
if folderFs.Type() == fs.FilesystemTypeBasic && !filepath.IsAbs(params["fsPath"]) {
uri = filepath.Join(folderFs.URI(), params["fsPath"])
}
versionsFs = fs.NewFilesystem(folderFs.Type(), uri)
} else {
var fsType fs.FilesystemType
_ = fsType.UnmarshalText([]byte(params["fsType"]))
versionsFs = fs.NewFilesystem(fsType, params["fsPath"])
}
l.Debugln("%s (%s) folder using %s (%s) versioner dir", folderFs.URI(), folderFs.Type(), versionsFs.URI(), versionsFs.Type())
return
}

4
lib/versioner/versioner.go
View File

@@ -9,6 +9,7 @@
package versioner
import (
"fmt"
"time"
"github.com/syncthing/syncthing/lib/fs"
@@ -16,6 +17,8 @@ import (
type Versioner interface {
Archive(filePath string) error
GetVersions() (map[string][]FileVersion, error)
Restore(filePath string, versionTime time.Time) error
}
type FileVersion struct {
@@ -25,6 +28,7 @@ type FileVersion struct {
}
var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
var ErrRestorationNotSupported = fmt.Errorf("version restoration not supported with the current versioner")
const (
TimeFormat = "20060102-150405"