all: Convert folders to use filesystem abstraction
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
committed by
Jakob Borg
parent
ab8c2fb5c7
commit
3d8b4a42b7
@@ -10,10 +10,11 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -23,15 +24,15 @@ func init() {
|
||||
|
||||
type External struct {
|
||||
command string
|
||||
folderPath string
|
||||
filesystem fs.Filesystem
|
||||
}
|
||||
|
||||
func NewExternal(folderID, folderPath string, params map[string]string) Versioner {
|
||||
func NewExternal(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner {
|
||||
command := params["command"]
|
||||
|
||||
s := External{
|
||||
command: command,
|
||||
folderPath: folderPath,
|
||||
filesystem: filesystem,
|
||||
}
|
||||
|
||||
l.Debugf("instantiated %#v", s)
|
||||
@@ -41,29 +42,41 @@ func NewExternal(folderID, folderPath string, params map[string]string) Versione
|
||||
// Archive moves the named file away to a version archive. If this function
|
||||
// returns nil, the named file does not exist any more (has been archived).
|
||||
func (v External) Archive(filePath string) error {
|
||||
info, err := osutil.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
info, err := v.filesystem.Lstat(filePath)
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("not archiving nonexistent file", filePath)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsSymlink() {
|
||||
panic("bug: attempting to version a symlink")
|
||||
}
|
||||
|
||||
l.Debugln("archiving", filePath)
|
||||
|
||||
inFolderPath, err := filepath.Rel(v.folderPath, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if v.command == "" {
|
||||
return errors.New("Versioner: command is empty, please enter a valid command")
|
||||
}
|
||||
|
||||
cmd := exec.Command(v.command, v.folderPath, inFolderPath)
|
||||
words, err := shellquote.Split(v.command)
|
||||
if err != nil {
|
||||
return errors.New("Versioner: command is invalid: " + err.Error())
|
||||
}
|
||||
|
||||
context := map[string]string{
|
||||
"%FOLDER_FILESYSTEM%": v.filesystem.Type().String(),
|
||||
"%FOLDER_PATH%": v.filesystem.URI(),
|
||||
"%FILE_PATH%": filePath,
|
||||
}
|
||||
|
||||
for i, word := range words {
|
||||
if replacement, ok := context[word]; ok {
|
||||
words[i] = replacement
|
||||
}
|
||||
}
|
||||
|
||||
cmd := exec.Command(words[0], words[1:]...)
|
||||
env := os.Environ()
|
||||
// filter STGUIAUTH and STGUIAPIKEY from environment variables
|
||||
filteredEnv := []string{}
|
||||
@@ -73,13 +86,14 @@ func (v External) Archive(filePath string) error {
|
||||
}
|
||||
}
|
||||
cmd.Env = filteredEnv
|
||||
err = cmd.Run()
|
||||
combinedOutput, err := cmd.CombinedOutput()
|
||||
l.Debugln("external command output:", string(combinedOutput))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// return error if the file was not removed
|
||||
if _, err = osutil.Lstat(filePath); os.IsNotExist(err) {
|
||||
if _, err = v.filesystem.Lstat(filePath); fs.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.New("Versioner: file was not removed by external script")
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
)
|
||||
|
||||
func TestExternalNoCommand(t *testing.T) {
|
||||
@@ -28,8 +30,8 @@ func TestExternalNoCommand(t *testing.T) {
|
||||
// The versioner should fail due to missing command.
|
||||
|
||||
e := External{
|
||||
filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
|
||||
command: "nonexistent command",
|
||||
folderPath: "testdata/folder path",
|
||||
}
|
||||
if err := e.Archive(file); err == nil {
|
||||
t.Error("Command should have failed")
|
||||
@@ -43,12 +45,12 @@ func TestExternalNoCommand(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestExternal(t *testing.T) {
|
||||
cmd := "./_external_test/external.sh"
|
||||
cmd := "./_external_test/external.sh %FOLDER_PATH% %FILE_PATH%"
|
||||
if runtime.GOOS == "windows" {
|
||||
cmd = `.\_external_test\external.bat`
|
||||
cmd = `.\\_external_test\\external.bat %FOLDER_PATH% %FILE_PATH%`
|
||||
}
|
||||
|
||||
file := "testdata/folder path/dir (parens)/long filename (parens).txt"
|
||||
file := filepath.Join("testdata", "folder path", "dir (parens)", "/long filename (parens).txt")
|
||||
prepForRemoval(t, file)
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
@@ -61,8 +63,8 @@ func TestExternal(t *testing.T) {
|
||||
// The versioner should run successfully.
|
||||
|
||||
e := External{
|
||||
filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "."),
|
||||
command: cmd,
|
||||
folderPath: "testdata/folder path",
|
||||
}
|
||||
if err := e.Archive(file); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -7,10 +7,10 @@
|
||||
package versioner
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/util"
|
||||
)
|
||||
@@ -21,19 +21,19 @@ func init() {
|
||||
}
|
||||
|
||||
type Simple struct {
|
||||
keep int
|
||||
folderPath string
|
||||
keep int
|
||||
fs fs.Filesystem
|
||||
}
|
||||
|
||||
func NewSimple(folderID, folderPath string, params map[string]string) Versioner {
|
||||
func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
|
||||
keep, err := strconv.Atoi(params["keep"])
|
||||
if err != nil {
|
||||
keep = 5 // A reasonable default
|
||||
}
|
||||
|
||||
s := Simple{
|
||||
keep: keep,
|
||||
folderPath: folderPath,
|
||||
keep: keep,
|
||||
fs: fs,
|
||||
}
|
||||
|
||||
l.Debugf("instantiated %#v", s)
|
||||
@@ -43,24 +43,24 @@ func NewSimple(folderID, folderPath string, params map[string]string) Versioner
|
||||
// Archive moves the named file away to a version archive. If this function
|
||||
// returns nil, the named file does not exist any more (has been archived).
|
||||
func (v Simple) Archive(filePath string) error {
|
||||
fileInfo, err := osutil.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
info, err := v.fs.Lstat(filePath)
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("not archiving nonexistent file", filePath)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if fileInfo.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsSymlink() {
|
||||
panic("bug: attempting to version a symlink")
|
||||
}
|
||||
|
||||
versionsDir := filepath.Join(v.folderPath, ".stversions")
|
||||
_, err = os.Stat(versionsDir)
|
||||
versionsDir := ".stversions"
|
||||
_, err = v.fs.Stat(versionsDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
l.Debugln("creating versions dir", versionsDir)
|
||||
osutil.MkdirAll(versionsDir, 0755)
|
||||
osutil.HideFile(versionsDir)
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("creating versions dir .stversions")
|
||||
v.fs.Mkdir(versionsDir, 0755)
|
||||
v.fs.Hide(versionsDir)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
@@ -69,28 +69,25 @@ func (v Simple) Archive(filePath string) error {
|
||||
l.Debugln("archiving", filePath)
|
||||
|
||||
file := filepath.Base(filePath)
|
||||
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inFolderPath := filepath.Dir(filePath)
|
||||
|
||||
dir := filepath.Join(versionsDir, inFolderPath)
|
||||
err = osutil.MkdirAll(dir, 0755)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
err = v.fs.MkdirAll(dir, 0755)
|
||||
if err != nil && !fs.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
ver := taggedFilename(file, fileInfo.ModTime().Format(TimeFormat))
|
||||
ver := taggedFilename(file, info.ModTime().Format(TimeFormat))
|
||||
dst := filepath.Join(dir, ver)
|
||||
l.Debugln("moving to", dst)
|
||||
err = osutil.Rename(filePath, dst)
|
||||
err = osutil.Rename(v.fs, filePath, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Glob according to the new file~timestamp.ext pattern.
|
||||
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
|
||||
newVersions, err := osutil.Glob(pattern)
|
||||
newVersions, err := v.fs.Glob(pattern)
|
||||
if err != nil {
|
||||
l.Warnln("globbing:", err, "for", pattern)
|
||||
return nil
|
||||
@@ -98,7 +95,7 @@ func (v Simple) Archive(filePath string) error {
|
||||
|
||||
// Also according to the old file.ext~timestamp pattern.
|
||||
pattern = filepath.Join(dir, file+"~"+TimeGlob)
|
||||
oldVersions, err := osutil.Glob(pattern)
|
||||
oldVersions, err := v.fs.Glob(pattern)
|
||||
if err != nil {
|
||||
l.Warnln("globbing:", err, "for", pattern)
|
||||
return nil
|
||||
@@ -111,7 +108,7 @@ func (v Simple) Archive(filePath string) error {
|
||||
if len(versions) > v.keep {
|
||||
for _, toRemove := range versions[:len(versions)-v.keep] {
|
||||
l.Debugln("cleaning out", toRemove)
|
||||
err = os.Remove(toRemove)
|
||||
err = v.fs.Remove(toRemove)
|
||||
if err != nil {
|
||||
l.Warnln("removing old version:", err)
|
||||
}
|
||||
|
||||
@@ -9,10 +9,11 @@ package versioner
|
||||
import (
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
)
|
||||
|
||||
func TestTaggedFilename(t *testing.T) {
|
||||
@@ -53,29 +54,28 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
|
||||
}
|
||||
|
||||
dir, err := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(dir)
|
||||
//defer os.RemoveAll(dir)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
v := NewSimple("", dir, map[string]string{"keep": "2"})
|
||||
versionDir := filepath.Join(dir, ".stversions")
|
||||
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
|
||||
|
||||
path := filepath.Join(dir, "test")
|
||||
v := NewSimple("", fs, map[string]string{"keep": "2"})
|
||||
|
||||
path := "test"
|
||||
|
||||
for i := 1; i <= 3; i++ {
|
||||
f, err := os.Create(path)
|
||||
f, err := fs.Create(path)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
f.Close()
|
||||
v.Archive(path)
|
||||
|
||||
d, err := os.Open(versionDir)
|
||||
if err != nil {
|
||||
if err := v.Archive(path); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
n, err := d.Readdirnames(-1)
|
||||
|
||||
n, err := fs.DirNames(".stversions")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -83,7 +83,6 @@ func TestSimpleVersioningVersionCount(t *testing.T) {
|
||||
if float64(len(n)) != math.Min(float64(i), 2) {
|
||||
t.Error("Wrong count")
|
||||
}
|
||||
d.Close()
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/util"
|
||||
)
|
||||
@@ -28,9 +28,9 @@ type Interval struct {
|
||||
}
|
||||
|
||||
type Staggered struct {
|
||||
versionsPath string
|
||||
cleanInterval int64
|
||||
folderPath string
|
||||
folderFs fs.Filesystem
|
||||
versionsFs fs.Filesystem
|
||||
interval [4]Interval
|
||||
mutex sync.Mutex
|
||||
|
||||
@@ -38,7 +38,7 @@ type Staggered struct {
|
||||
testCleanDone chan struct{}
|
||||
}
|
||||
|
||||
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
|
||||
func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
|
||||
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
|
||||
if err != nil {
|
||||
maxAge = 31536000 // Default: ~1 year
|
||||
@@ -49,22 +49,20 @@ func NewStaggered(folderID, folderPath string, params map[string]string) Version
|
||||
}
|
||||
|
||||
// Use custom path if set, otherwise .stversions in folderPath
|
||||
var versionsDir string
|
||||
var versionsFs fs.Filesystem
|
||||
if params["versionsPath"] == "" {
|
||||
versionsDir = filepath.Join(folderPath, ".stversions")
|
||||
l.Debugln("using default dir .stversions")
|
||||
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
|
||||
} else if filepath.IsAbs(params["versionsPath"]) {
|
||||
l.Debugln("using dir", params["versionsPath"])
|
||||
versionsDir = params["versionsPath"]
|
||||
versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
|
||||
} else {
|
||||
versionsDir = filepath.Join(folderPath, params["versionsPath"])
|
||||
l.Debugln("using dir", versionsDir)
|
||||
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
|
||||
}
|
||||
l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
|
||||
|
||||
s := &Staggered{
|
||||
versionsPath: versionsDir,
|
||||
cleanInterval: cleanInterval,
|
||||
folderPath: folderPath,
|
||||
folderFs: folderFs,
|
||||
versionsFs: versionsFs,
|
||||
interval: [4]Interval{
|
||||
{30, 3600}, // first hour -> 30 sec between versions
|
||||
{3600, 86400}, // next day -> 1 h between versions
|
||||
@@ -102,12 +100,12 @@ func (v *Staggered) Stop() {
|
||||
}
|
||||
|
||||
func (v *Staggered) clean() {
|
||||
l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
|
||||
l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
|
||||
v.mutex.Lock()
|
||||
defer v.mutex.Unlock()
|
||||
l.Debugln("Versioner clean: Cleaning", v.versionsPath)
|
||||
l.Debugln("Versioner clean: Cleaning", v.versionsFs)
|
||||
|
||||
if _, err := os.Stat(v.versionsPath); os.IsNotExist(err) {
|
||||
if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
|
||||
// There is no need to clean a nonexistent dir.
|
||||
return
|
||||
}
|
||||
@@ -115,14 +113,14 @@ func (v *Staggered) clean() {
|
||||
versionsPerFile := make(map[string][]string)
|
||||
filesPerDir := make(map[string]int)
|
||||
|
||||
err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
|
||||
err := v.versionsFs.Walk(".", func(path string, f fs.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
|
||||
if f.IsDir() && !f.IsSymlink() {
|
||||
filesPerDir[path] = 0
|
||||
if path != v.versionsPath {
|
||||
if path != "." {
|
||||
dir := filepath.Dir(path)
|
||||
filesPerDir[dir]++
|
||||
}
|
||||
@@ -155,25 +153,20 @@ func (v *Staggered) clean() {
|
||||
continue
|
||||
}
|
||||
|
||||
if path == v.versionsPath {
|
||||
l.Debugln("Cleaner: versions dir is empty, don't delete", path)
|
||||
continue
|
||||
}
|
||||
|
||||
l.Debugln("Cleaner: deleting empty directory", path)
|
||||
err = os.Remove(path)
|
||||
err = v.versionsFs.Remove(path)
|
||||
if err != nil {
|
||||
l.Warnln("Versioner: can't remove directory", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
|
||||
l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
|
||||
}
|
||||
|
||||
func (v *Staggered) expire(versions []string) {
|
||||
l.Debugln("Versioner: Expiring versions", versions)
|
||||
for _, file := range v.toRemove(versions, time.Now()) {
|
||||
if fi, err := osutil.Lstat(file); err != nil {
|
||||
if fi, err := v.versionsFs.Lstat(file); err != nil {
|
||||
l.Warnln("versioner:", err)
|
||||
continue
|
||||
} else if fi.IsDir() {
|
||||
@@ -181,7 +174,7 @@ func (v *Staggered) expire(versions []string) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.Remove(file); err != nil {
|
||||
if err := v.versionsFs.Remove(file); err != nil {
|
||||
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
||||
}
|
||||
}
|
||||
@@ -203,7 +196,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
|
||||
// If the file is older than the max age of the last interval, remove it
|
||||
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
|
||||
l.Debugln("Versioner: File over maximum age -> delete ", file)
|
||||
err = os.Remove(file)
|
||||
err = v.versionsFs.Remove(file)
|
||||
if err != nil {
|
||||
l.Warnf("Versioner: can't remove %q: %v", file, err)
|
||||
}
|
||||
@@ -240,26 +233,26 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
|
||||
// Archive moves the named file away to a version archive. If this function
|
||||
// returns nil, the named file does not exist any more (has been archived).
|
||||
func (v *Staggered) Archive(filePath string) error {
|
||||
l.Debugln("Waiting for lock on ", v.versionsPath)
|
||||
l.Debugln("Waiting for lock on ", v.versionsFs)
|
||||
v.mutex.Lock()
|
||||
defer v.mutex.Unlock()
|
||||
|
||||
info, err := osutil.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
info, err := v.folderFs.Lstat(filePath)
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("not archiving nonexistent file", filePath)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsSymlink() {
|
||||
panic("bug: attempting to version a symlink")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(v.versionsPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
l.Debugln("creating versions dir", v.versionsPath)
|
||||
osutil.MkdirAll(v.versionsPath, 0755)
|
||||
osutil.HideFile(v.versionsPath)
|
||||
if _, err := v.versionsFs.Stat("."); err != nil {
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("creating versions dir", v.versionsFs)
|
||||
v.versionsFs.MkdirAll(".", 0755)
|
||||
v.versionsFs.Hide(".")
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
@@ -268,36 +261,41 @@ func (v *Staggered) Archive(filePath string) error {
|
||||
l.Debugln("archiving", filePath)
|
||||
|
||||
file := filepath.Base(filePath)
|
||||
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
|
||||
inFolderPath := filepath.Dir(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir := filepath.Join(v.versionsPath, inFolderPath)
|
||||
err = osutil.MkdirAll(dir, 0755)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
err = v.versionsFs.MkdirAll(inFolderPath, 0755)
|
||||
if err != nil && !fs.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
ver := taggedFilename(file, time.Now().Format(TimeFormat))
|
||||
dst := filepath.Join(dir, ver)
|
||||
dst := filepath.Join(inFolderPath, ver)
|
||||
l.Debugln("moving to", dst)
|
||||
err = osutil.Rename(filePath, dst)
|
||||
|
||||
/// TODO: Fix this when we have an alternative filesystem implementation
|
||||
if v.versionsFs.Type() != fs.FilesystemTypeBasic {
|
||||
panic("bug: staggered versioner used with unsupported filesystem")
|
||||
}
|
||||
|
||||
err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Glob according to the new file~timestamp.ext pattern.
|
||||
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
|
||||
newVersions, err := osutil.Glob(pattern)
|
||||
pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob))
|
||||
newVersions, err := v.versionsFs.Glob(pattern)
|
||||
if err != nil {
|
||||
l.Warnln("globbing:", err, "for", pattern)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Also according to the old file.ext~timestamp pattern.
|
||||
pattern = filepath.Join(dir, file+"~"+TimeGlob)
|
||||
oldVersions, err := osutil.Glob(pattern)
|
||||
pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob)
|
||||
oldVersions, err := v.versionsFs.Glob(pattern)
|
||||
if err != nil {
|
||||
l.Warnln("globbing:", err, "for", pattern)
|
||||
return nil
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/d4l3k/messagediff"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
)
|
||||
|
||||
func TestStaggeredVersioningVersionCount(t *testing.T) {
|
||||
@@ -62,7 +63,7 @@ func TestStaggeredVersioningVersionCount(t *testing.T) {
|
||||
os.MkdirAll("testdata/.stversions", 0755)
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
v := NewStaggered("", "testdata", map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
|
||||
v := NewStaggered("", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"maxAge": strconv.Itoa(365 * 86400)}).(*Staggered)
|
||||
v.testCleanDone = make(chan struct{})
|
||||
defer v.Stop()
|
||||
go v.Serve()
|
||||
|
||||
@@ -8,11 +8,11 @@ package versioner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
)
|
||||
|
||||
@@ -22,17 +22,17 @@ func init() {
|
||||
}
|
||||
|
||||
type Trashcan struct {
|
||||
folderPath string
|
||||
fs fs.Filesystem
|
||||
cleanoutDays int
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
|
||||
func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
|
||||
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
|
||||
// On error we default to 0, "do not clean out the trash can"
|
||||
|
||||
s := &Trashcan{
|
||||
folderPath: folderPath,
|
||||
fs: fs,
|
||||
cleanoutDays: cleanoutDays,
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
@@ -44,52 +44,47 @@ func NewTrashcan(folderID, folderPath string, params map[string]string) Versione
|
||||
// Archive moves the named file away to a version archive. If this function
|
||||
// returns nil, the named file does not exist any more (has been archived).
|
||||
func (t *Trashcan) Archive(filePath string) error {
|
||||
info, err := osutil.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
info, err := t.fs.Lstat(filePath)
|
||||
if fs.IsNotExist(err) {
|
||||
l.Debugln("not archiving nonexistent file", filePath)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info.IsSymlink() {
|
||||
panic("bug: attempting to version a symlink")
|
||||
}
|
||||
|
||||
versionsDir := filepath.Join(t.folderPath, ".stversions")
|
||||
if _, err := os.Stat(versionsDir); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
versionsDir := ".stversions"
|
||||
if _, err := t.fs.Stat(versionsDir); err != nil {
|
||||
if !fs.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
l.Debugln("creating versions dir", versionsDir)
|
||||
if err := osutil.MkdirAll(versionsDir, 0777); err != nil {
|
||||
if err := t.fs.MkdirAll(versionsDir, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
osutil.HideFile(versionsDir)
|
||||
t.fs.Hide(versionsDir)
|
||||
}
|
||||
|
||||
l.Debugln("archiving", filePath)
|
||||
|
||||
relativePath, err := filepath.Rel(t.folderPath, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
archivedPath := filepath.Join(versionsDir, relativePath)
|
||||
if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) {
|
||||
archivedPath := filepath.Join(versionsDir, filePath)
|
||||
if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
l.Debugln("moving to", archivedPath)
|
||||
|
||||
if err := osutil.Rename(filePath, archivedPath); err != nil {
|
||||
if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set the mtime to the time the file was deleted. This is used by the
|
||||
// cleanout routine. If this fails things won't work optimally but there's
|
||||
// not much we can do about it so we ignore the error.
|
||||
os.Chtimes(archivedPath, time.Now(), time.Now())
|
||||
t.fs.Chtimes(archivedPath, time.Now(), time.Now())
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -129,15 +124,15 @@ func (t *Trashcan) String() string {
|
||||
}
|
||||
|
||||
func (t *Trashcan) cleanoutArchive() error {
|
||||
versionsDir := filepath.Join(t.folderPath, ".stversions")
|
||||
if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) {
|
||||
versionsDir := ".stversions"
|
||||
if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour)
|
||||
currentDir := ""
|
||||
filesInDir := 0
|
||||
walkFn := func(path string, info os.FileInfo, err error) error {
|
||||
walkFn := func(path string, info fs.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -147,7 +142,7 @@ func (t *Trashcan) cleanoutArchive() error {
|
||||
// directory was empty and try to remove it. We ignore failure for
|
||||
// the time being.
|
||||
if currentDir != "" && filesInDir == 0 {
|
||||
os.Remove(currentDir)
|
||||
t.fs.Remove(currentDir)
|
||||
}
|
||||
currentDir = path
|
||||
filesInDir = 0
|
||||
@@ -156,7 +151,7 @@ func (t *Trashcan) cleanoutArchive() error {
|
||||
|
||||
if info.ModTime().Before(cutoff) {
|
||||
// The file is too old; remove it.
|
||||
os.Remove(path)
|
||||
t.fs.Remove(path)
|
||||
} else {
|
||||
// Keep this file, and remember it so we don't unnecessarily try
|
||||
// to remove this directory.
|
||||
@@ -165,14 +160,14 @@ func (t *Trashcan) cleanoutArchive() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := filepath.Walk(versionsDir, walkFn); err != nil {
|
||||
if err := t.fs.Walk(versionsDir, walkFn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The last directory seen by the walkFn may not have been removed as it
|
||||
// should be.
|
||||
if currentDir != "" && filesInDir == 0 {
|
||||
os.Remove(currentDir)
|
||||
t.fs.Remove(currentDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
)
|
||||
|
||||
func TestTrashcanCleanout(t *testing.T) {
|
||||
@@ -49,7 +51,7 @@ func TestTrashcanCleanout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan)
|
||||
versioner := NewTrashcan("default", fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"), map[string]string{"cleanoutDays": "7"}).(*Trashcan)
|
||||
if err := versioner.cleanoutArchive(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@
|
||||
// simple default versioning scheme.
|
||||
package versioner
|
||||
|
||||
import "github.com/syncthing/syncthing/lib/fs"
|
||||
|
||||
type Versioner interface {
|
||||
Archive(filePath string) error
|
||||
}
|
||||
|
||||
var Factories = map[string]func(folderID string, folderDir string, params map[string]string) Versioner{}
|
||||
var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
|
||||
|
||||
const (
|
||||
TimeFormat = "20060102-150405"
|
||||
|
||||
Reference in New Issue
Block a user