cmd/syncthing: UI for version restoration (fixes #2599) (#4602)

cmd/syncthing: Add UI for version restoration (fixes #2599)

lib/model/model.go

@@ -38,6 +38,16 @@ import (
 	"github.com/thejerf/suture"
 )
 
+var locationLocal *time.Location
+
+func init() {
+	var err error
+	locationLocal, err = time.LoadLocation("Local")
+	if err != nil {
+		panic(err.Error())
+	}
+}
+
 // How many files to send in each Index/IndexUpdate message.
 const (
 	maxBatchSizeBytes = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
@@ -232,21 +242,13 @@ func (m *Model) startFolderLocked(folder string) config.FolderType {
 		}
 	}
 
-	var ver versioner.Versioner
-	if len(cfg.Versioning.Type) > 0 {
-		versionerFactory, ok := versioner.Factories[cfg.Versioning.Type]
-		if !ok {
-			l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
-		}
-
-		ver = versionerFactory(folder, cfg.Filesystem(), cfg.Versioning.Params)
-		if service, ok := ver.(suture.Service); ok {
-			// The versioner implements the suture.Service interface, so
-			// expects to be run in the background in addition to being called
-			// when files are going to be archived.
-			token := m.Add(service)
-			m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
-		}
+	ver := cfg.Versioner()
+	if service, ok := ver.(suture.Service); ok {
+		// The versioner implements the suture.Service interface, so
+		// expects to be run in the background in addition to being called
+		// when files are going to be archived.
+		token := m.Add(service)
+		m.folderRunnerTokens[folder] = append(m.folderRunnerTokens[folder], token)
 	}
 
 	ffs := fs.MtimeFS()
@@ -2376,6 +2378,132 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 	return output
 }
 
+func (m *Model) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
+	fcfg, ok := m.cfg.Folder(folder)
+	if !ok {
+		return nil, errFolderMissing
+	}
+
+	files := make(map[string][]versioner.FileVersion)
+
+	filesystem := fcfg.Filesystem()
+	err := filesystem.Walk(".stversions", func(path string, f fs.FileInfo, err error) error {
+		// Skip root (which is ok to be a symlink)
+		if path == ".stversions" {
+			return nil
+		}
+
+		// Ignore symlinks
+		if f.IsSymlink() {
+			return fs.SkipDir
+		}
+
+		// No records for directories
+		if f.IsDir() {
+			return nil
+		}
+
+		// Strip .stversions prefix.
+		path = strings.TrimPrefix(path, ".stversions"+string(fs.PathSeparator))
+
+		name, tag := versioner.UntagFilename(path)
+		// Something invalid
+		if name == "" || tag == "" {
+			return nil
+		}
+
+		name = osutil.NormalizedFilename(name)
+
+		versionTime, err := time.ParseInLocation(versioner.TimeFormat, tag, locationLocal)
+		if err != nil {
+			return nil
+		}
+
+		files[name] = append(files[name], versioner.FileVersion{
+			VersionTime: versionTime.Truncate(time.Second),
+			ModTime:     f.ModTime().Truncate(time.Second),
+			Size:        f.Size(),
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}
+
+func (m *Model) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
+	fcfg, ok := m.cfg.Folder(folder)
+	if !ok {
+		return nil, errFolderMissing
+	}
+
+	filesystem := fcfg.Filesystem()
+	ver := fcfg.Versioner()
+
+	restore := make(map[string]string)
+	errors := make(map[string]string)
+
+	// Validation
+	for file, version := range versions {
+		file = osutil.NativeFilename(file)
+		tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat)
+		versionedTaggedFilename := filepath.Join(".stversions", versioner.TagFilename(file, tag))
+		// Check that the thing we've been asked to restore is actually a file
+		// and that it exists.
+		if info, err := filesystem.Lstat(versionedTaggedFilename); err != nil {
+			errors[file] = err.Error()
+			continue
+		} else if !info.IsRegular() {
+			errors[file] = "not a file"
+			continue
+		}
+
+		// Check that the target location of where we are supposed to restore
+		// either does not exist, or is actually a file.
+		if info, err := filesystem.Lstat(file); err == nil && !info.IsRegular() {
+			errors[file] = "cannot replace a non-file"
+			continue
+		} else if err != nil && !fs.IsNotExist(err) {
+			errors[file] = err.Error()
+			continue
+		}
+
+		restore[file] = versionedTaggedFilename
+	}
+
+	// Execution
+	var err error
+	for target, source := range restore {
+		err = nil
+		if _, serr := filesystem.Lstat(target); serr == nil {
+			if ver != nil {
+				err = osutil.InWritableDir(ver.Archive, filesystem, target)
+			} else {
+				err = osutil.InWritableDir(filesystem.Remove, filesystem, target)
+			}
+		}
+
+		filesystem.MkdirAll(filepath.Dir(target), 0755)
+		if err == nil {
+			err = osutil.Copy(filesystem, source, target)
+		}
+
+		if err != nil {
+			errors[target] = err.Error()
+			continue
+		}
+	}
+
+	// Trigger scan
+	if !fcfg.FSWatcherEnabled {
+		m.ScanFolder(folder)
+	}
+
+	return errors, nil
+}
+
 func (m *Model) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []Availability {
 	// The slightly unusual locking sequence here is because we need to hold
 	// pmut for the duration (as the value returned from foldersFiles can

lib/model/model_test.go

@@ -29,9 +29,11 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	srand "github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/scanner"
+	"github.com/syncthing/syncthing/lib/versioner"
 )
 
 var device1, device2 protocol.DeviceID
@@ -2871,6 +2873,217 @@ func TestIssue4475(t *testing.T) {
 	}
 }
 
+func TestVersionRestore(t *testing.T) {
+	// We create a bunch of files which we restore
+	// In each file, we write the filename as the content
+	// We verify that the content matches at the expected filenames
+	// after the restore operation.
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	dbi := db.OpenMemory()
+
+	fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, dir)
+	fcfg.Versioning.Type = "simple"
+	fcfg.FSWatcherEnabled = false
+	filesystem := fcfg.Filesystem()
+
+	rawConfig := config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	}
+	cfg := config.Wrap("/tmp/test", rawConfig)
+
+	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", dbi, nil)
+	m.AddFolder(fcfg)
+	m.StartFolder("default")
+	m.ServeBackground()
+	defer m.Stop()
+	m.ScanFolder("default")
+
+	sentinel, err := time.ParseInLocation(versioner.TimeFormat, "20200101-010101", locationLocal)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sentinelTag := sentinel.Format(versioner.TimeFormat)
+
+	for _, file := range []string{
+		// Versions directory
+		".stversions/file~20171210-040404.txt",  // will be restored
+		".stversions/existing~20171210-040404",  // exists, should expect to be archived.
+		".stversions/something~20171210-040404", // will become directory, hence error
+		".stversions/dir/file~20171210-040404.txt",
+		".stversions/dir/file~20171210-040405.txt",
+		".stversions/dir/file~20171210-040406.txt",
+		".stversions/very/very/deep/one~20171210-040406.txt", // lives deep down, no directory exists.
+		".stversions/dir/existing~20171210-040406.txt",       // exists, should expect to be archived.
+		".stversions/dir/file.txt~20171210-040405",           // incorrect tag format, ignored.
+		".stversions/dir/cat",                                // incorrect tag format, ignored.
+
+		// "file.txt" will be restored
+		"existing",
+		"something/file", // Becomes directory
+		"dir/file.txt",
+		"dir/existing.txt",
+	} {
+		if runtime.GOOS == "windows" {
+			file = filepath.FromSlash(file)
+		}
+		dir := filepath.Dir(file)
+		if err := filesystem.MkdirAll(dir, 0755); err != nil {
+			t.Fatal(err)
+		}
+		if fd, err := filesystem.Create(file); err != nil {
+			t.Fatal(err)
+		} else if _, err := fd.Write([]byte(file)); err != nil {
+			t.Fatal(err)
+		} else if err := fd.Close(); err != nil {
+			t.Fatal(err)
+		} else if err := filesystem.Chtimes(file, sentinel, sentinel); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	versions, err := m.GetFolderVersions("default")
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedVersions := map[string]int{
+		"file.txt":               1,
+		"existing":               1,
+		"something":              1,
+		"dir/file.txt":           3,
+		"dir/existing.txt":       1,
+		"very/very/deep/one.txt": 1,
+	}
+
+	for name, vers := range versions {
+		cnt, ok := expectedVersions[name]
+		if !ok {
+			t.Errorf("unexpected %s", name)
+		}
+		if len(vers) != cnt {
+			t.Errorf("%s: %s != %s", name, cnt, len(vers))
+		}
+		// Delete, so we can check if we didn't hit something we expect afterwards.
+		delete(expectedVersions, name)
+	}
+
+	for name := range expectedVersions {
+		t.Errorf("not found expected %s", name)
+	}
+
+	// Restoring non existing folder fails.
+	_, err = m.RestoreFolderVersions("does not exist", nil)
+	if err == nil {
+		t.Errorf("expected an error")
+	}
+
+	makeTime := func(s string) time.Time {
+		tm, err := time.ParseInLocation(versioner.TimeFormat, s, locationLocal)
+		if err != nil {
+			t.Error(err)
+		}
+		return tm.Truncate(time.Second)
+	}
+
+	restore := map[string]time.Time{
+		"file.txt":               makeTime("20171210-040404"),
+		"existing":               makeTime("20171210-040404"),
+		"something":              makeTime("20171210-040404"),
+		"dir/file.txt":           makeTime("20171210-040406"),
+		"dir/existing.txt":       makeTime("20171210-040406"),
+		"very/very/deep/one.txt": makeTime("20171210-040406"),
+	}
+
+	ferr, err := m.RestoreFolderVersions("default", restore)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot replace a non-file" {
+		t.Fatalf("incorrect error or count: %d %s", len(ferr), ferr)
+	}
+
+	// Failed items are not expected to be restored.
+	// Remove them from expectations
+	for name := range ferr {
+		delete(restore, name)
+	}
+
+	// Check that content of files matches to the version they've been restored.
+	for file, version := range restore {
+		if runtime.GOOS == "windows" {
+			file = filepath.FromSlash(file)
+		}
+		tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat)
+		taggedName := filepath.Join(".stversions", versioner.TagFilename(file, tag))
+		fd, err := filesystem.Open(file)
+		if err != nil {
+			t.Error(err)
+		}
+		defer fd.Close()
+
+		content, err := ioutil.ReadAll(fd)
+		if err != nil {
+			t.Error(err)
+		}
+		if !bytes.Equal(content, []byte(taggedName)) {
+			t.Errorf("%s: %s != %s", file, string(content), taggedName)
+		}
+	}
+
+	// Simple versioner uses modtime for timestamp generation, so we can check
+	// if existing stuff was correctly archived as we restored.
+	expectArchived := map[string]struct{}{
+		"existing":         {},
+		"dir/file.txt":     {},
+		"dir/existing.txt": {},
+	}
+
+	// Even if they are at the archived path, content should have the non
+	// archived name.
+	for file := range expectArchived {
+		if runtime.GOOS == "windows" {
+			file = filepath.FromSlash(file)
+		}
+		taggedName := versioner.TagFilename(file, sentinelTag)
+		taggedArchivedName := filepath.Join(".stversions", taggedName)
+
+		fd, err := filesystem.Open(taggedArchivedName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer fd.Close()
+
+		content, err := ioutil.ReadAll(fd)
+		if err != nil {
+			t.Error(err)
+		}
+		if !bytes.Equal(content, []byte(file)) {
+			t.Errorf("%s: %s != %s", file, string(content), file)
+		}
+	}
+
+	// Check for other unexpected things that are tagged.
+	filesystem.Walk(".", func(path string, f fs.FileInfo, err error) error {
+		if !f.IsRegular() {
+			return nil
+		}
+		if strings.Contains(path, sentinelTag) {
+			path = osutil.NormalizedFilename(path)
+			name, _ := versioner.UntagFilename(path)
+			name = strings.TrimPrefix(name, ".stversions/")
+			if _, ok := expectArchived[name]; !ok {
+				t.Errorf("unexpected file with sentinel tag: %s", name)
+			}
+		}
+		return nil
+	})
+}
+
 func TestPausedFolders(t *testing.T) {
 	// Create a separate wrapper not to pollute other tests.
 	cfg := defaultConfig.RawCopy()

lib/model/rwfolder.go

@@ -1451,7 +1451,7 @@ func (f *sendReceiveFolder) performFinish(ignores *ignore.Matcher, state *shared
 			// file before we replace it. Archiving a non-existent file is not
 			// an error.
 
-			if err = f.versioner.Archive(state.file.Name); err != nil {
+			if err = osutil.InWritableDir(f.versioner.Archive, f.fs, state.file.Name); err != nil {
 				return err
 			}
 		}