lib/versioner: Restore for all versioners, cross-device support (#5514)

* lib/versioner: Restore for all versioners, cross-device support

Fixes #4631
Fixes #4586
Fixes #1634
Fixes #5338
Fixes #5419

lib/osutil/osutil.go

@@ -22,37 +22,62 @@ import (
 // often enough that there is any contention on this lock.
 var renameLock = sync.NewMutex()
 
-// TryRename renames a file, leaving source file intact in case of failure.
+// RenameOrCopy renames a file, leaving source file intact in case of failure.
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func TryRename(filesystem fs.Filesystem, from, to string) error {
+func RenameOrCopy(src, dst fs.Filesystem, from, to string) error {
 	renameLock.Lock()
 	defer renameLock.Unlock()
 
-	return withPreparedTarget(filesystem, from, to, func() error {
-		return filesystem.Rename(from, to)
-	})
-}
+	return withPreparedTarget(dst, from, to, func() error {
+		// Optimisation 1
+		if src.Type() == dst.Type() && src.URI() == dst.URI() {
+			return src.Rename(from, to)
+		}
 
-// Rename moves a temporary file to its final place.
-// Will make sure to delete the from file if the operation fails, so use only
-// for situations like committing a temp file to its final location.
-// Tries hard to succeed on various systems by temporarily tweaking directory
-// permissions and removing the destination file when necessary.
-func Rename(filesystem fs.Filesystem, from, to string) error {
-	// Don't leave a dangling temp file in case of rename error
-	if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) {
-		defer filesystem.Remove(from)
-	}
-	return TryRename(filesystem, from, to)
+		// "Optimisation" 2
+		// Try to find a common prefix between the two filesystems, use that as the base for the new one
+		// and try a rename.
+		if src.Type() == dst.Type() {
+			commonPrefix := fs.CommonPrefix(src.URI(), dst.URI())
+			if len(commonPrefix) > 0 {
+				commonFs := fs.NewFilesystem(src.Type(), commonPrefix)
+				err := commonFs.Rename(
+					filepath.Join(strings.TrimPrefix(src.URI(), commonPrefix), from),
+					filepath.Join(strings.TrimPrefix(dst.URI(), commonPrefix), to),
+				)
+				if err == nil {
+					return nil
+				}
+			}
+		}
+
+		// Everything is sad, do a copy and delete.
+		if _, err := dst.Stat(to); !fs.IsNotExist(err) {
+			err := dst.Remove(to)
+			if err != nil {
+				return err
+			}
+		}
+
+		err := copyFileContents(src, dst, from, to)
+		if err != nil {
+			_ = dst.Remove(to)
+			return err
+		}
+
+		return withPreparedTarget(src, from, from, func() error {
+			return src.Remove(from)
+		})
+	})
 }
 
 // Copy copies the file content from source to destination.
 // Tries hard to succeed on various systems by temporarily tweaking directory
 // permissions and removing the destination file when necessary.
-func Copy(filesystem fs.Filesystem, from, to string) (err error) {
-	return withPreparedTarget(filesystem, from, to, func() error {
-		return copyFileContents(filesystem, from, to)
+func Copy(src, dst fs.Filesystem, from, to string) (err error) {
+	return withPreparedTarget(dst, from, to, func() error {
+		return copyFileContents(src, dst, from, to)
 	})
 }
 
@@ -115,13 +140,13 @@ func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() erro
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all its contents will be replaced by the contents
 // of the source file.
-func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) {
-	in, err := filesystem.Open(src)
+func copyFileContents(srcFs, dstFs fs.Filesystem, src, dst string) (err error) {
+	in, err := srcFs.Open(src)
 	if err != nil {
 		return
 	}
 	defer in.Close()
-	out, err := filesystem.Create(dst)
+	out, err := dstFs.Create(dst)
 	if err != nil {
 		return
 	}

lib/osutil/osutil_test.go

@@ -7,6 +7,7 @@
 package osutil_test
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -192,7 +193,7 @@ func TestInWritableDirWindowsRename(t *testing.T) {
 	}
 
 	rename := func(path string) error {
-		return osutil.Rename(fs, path, path+"new")
+		return osutil.RenameOrCopy(fs, fs, path, path+"new")
 	}
 
 	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
@@ -268,3 +269,79 @@ func TestIsDeleted(t *testing.T) {
 	testFs.Chmod("inacc", 0777)
 	os.RemoveAll("testdata")
 }
+
+func TestRenameOrCopy(t *testing.T) {
+	mustTempDir := func() string {
+		t.Helper()
+		tmpDir, err := ioutil.TempDir("", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return tmpDir
+	}
+	sameFs := fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir())
+	tests := []struct {
+		src  fs.Filesystem
+		dst  fs.Filesystem
+		file string
+	}{
+		{
+			src:  sameFs,
+			dst:  sameFs,
+			file: "file",
+		},
+		{
+			src:  fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
+			dst:  fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
+			file: "file",
+		},
+		{
+			src:  fs.NewFilesystem(fs.FilesystemTypeFake, `fake://fake/?files=1&seed=42`),
+			dst:  fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
+			file: osutil.NativeFilename(`05/7a/4d52f284145b9fe8`),
+		},
+	}
+
+	for _, test := range tests {
+		content := test.src.URI()
+		if _, err := test.src.Lstat(test.file); err != nil {
+			if !fs.IsNotExist(err) {
+				t.Fatal(err)
+			}
+			if fd, err := test.src.Create(test.file); err != nil {
+				t.Fatal(err)
+			} else {
+				if _, err := fd.Write([]byte(test.src.URI())); err != nil {
+					t.Fatal(err)
+				}
+				_ = fd.Close()
+			}
+		} else {
+			fd, err := test.src.Open(test.file)
+			if err != nil {
+				t.Fatal(err)
+			}
+			buf, err := ioutil.ReadAll(fd)
+			if err != nil {
+				t.Fatal(err)
+			}
+			_ = fd.Close()
+			content = string(buf)
+		}
+
+		err := osutil.RenameOrCopy(test.src, test.dst, test.file, "new")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if fd, err := test.dst.Open("new"); err != nil {
+			t.Fatal(err)
+		} else {
+			if buf, err := ioutil.ReadAll(fd); err != nil {
+				t.Fatal(err)
+			} else if string(buf) != content {
+				t.Fatalf("expected %s got %s", content, string(buf))
+			}
+		}
+	}
+}