perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Convert folders to use filesystem abstraction

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
Audrius Butkevicius
2017-08-19 14:36:56 +00:00
committed by Jakob Borg
parent ab8c2fb5c7
commit 3d8b4a42b7
78 changed files with 2588 additions and 1665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
lib/versioner/staggered.go
View File

@@ -12,7 +12,7 @@ import (
"strconv"
"time"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
@@ -28,9 +28,9 @@ type Interval struct {
}
type Staggered struct {
versionsPath string
cleanInterval int64
folderPath string
folderFs fs.Filesystem
versionsFs fs.Filesystem
interval [4]Interval
mutex sync.Mutex
@@ -38,7 +38,7 @@ type Staggered struct {
testCleanDone chan struct{}
}
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
if err != nil {
maxAge = 31536000 // Default: ~1 year
@@ -49,22 +49,20 @@ func NewStaggered(folderID, folderPath string, params map[string]string) Version
}
// Use custom path if set, otherwise .stversions in folderPath
var versionsDir string
var versionsFs fs.Filesystem
if params["versionsPath"] == "" {
versionsDir = filepath.Join(folderPath, ".stversions")
l.Debugln("using default dir .stversions")
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if filepath.IsAbs(params["versionsPath"]) {
l.Debugln("using dir", params["versionsPath"])
versionsDir = params["versionsPath"]
versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
} else {
versionsDir = filepath.Join(folderPath, params["versionsPath"])
l.Debugln("using dir", versionsDir)
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
}
l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
s := &Staggered{
versionsPath: versionsDir,
cleanInterval: cleanInterval,
folderPath: folderPath,
folderFs: folderFs,
versionsFs: versionsFs,
interval: [4]Interval{
{30, 3600}, // first hour -> 30 sec between versions
{3600, 86400}, // next day -> 1 h between versions
@@ -102,12 +100,12 @@ func (v *Staggered) Stop() {
}
func (v *Staggered) clean() {
l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
l.Debugln("Versioner clean: Waiting for lock on", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
l.Debugln("Versioner clean: Cleaning", v.versionsPath)
l.Debugln("Versioner clean: Cleaning", v.versionsFs)
if _, err := os.Stat(v.versionsPath); os.IsNotExist(err) {
if _, err := v.versionsFs.Stat("."); fs.IsNotExist(err) {
// There is no need to clean a nonexistent dir.
return
}
@@ -115,14 +113,14 @@ func (v *Staggered) clean() {
versionsPerFile := make(map[string][]string)
filesPerDir := make(map[string]int)
err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
err := v.versionsFs.Walk(".", func(path string, f fs.FileInfo, err error) error {
if err != nil {
return err
}
if f.Mode().IsDir() && f.Mode()&os.ModeSymlink == 0 {
if f.IsDir() && !f.IsSymlink() {
filesPerDir[path] = 0
if path != v.versionsPath {
if path != "." {
dir := filepath.Dir(path)
filesPerDir[dir]++
}
@@ -155,25 +153,20 @@ func (v *Staggered) clean() {
continue
}
if path == v.versionsPath {
l.Debugln("Cleaner: versions dir is empty, don't delete", path)
continue
}
l.Debugln("Cleaner: deleting empty directory", path)
err = os.Remove(path)
err = v.versionsFs.Remove(path)
if err != nil {
l.Warnln("Versioner: can't remove directory", path, err)
}
}
l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
l.Debugln("Cleaner: Finished cleaning", v.versionsFs)
}
func (v *Staggered) expire(versions []string) {
l.Debugln("Versioner: Expiring versions", versions)
for _, file := range v.toRemove(versions, time.Now()) {
if fi, err := osutil.Lstat(file); err != nil {
if fi, err := v.versionsFs.Lstat(file); err != nil {
l.Warnln("versioner:", err)
continue
} else if fi.IsDir() {
@@ -181,7 +174,7 @@ func (v *Staggered) expire(versions []string) {
continue
}
if err := os.Remove(file); err != nil {
if err := v.versionsFs.Remove(file); err != nil {
l.Warnf("Versioner: can't remove %q: %v", file, err)
}
}
@@ -203,7 +196,7 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
// If the file is older than the max age of the last interval, remove it
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
l.Debugln("Versioner: File over maximum age -> delete ", file)
err = os.Remove(file)
err = v.versionsFs.Remove(file)
if err != nil {
l.Warnf("Versioner: can't remove %q: %v", file, err)
}
@@ -240,26 +233,26 @@ func (v *Staggered) toRemove(versions []string, now time.Time) []string {
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v *Staggered) Archive(filePath string) error {
l.Debugln("Waiting for lock on ", v.versionsPath)
l.Debugln("Waiting for lock on ", v.versionsFs)
v.mutex.Lock()
defer v.mutex.Unlock()
info, err := osutil.Lstat(filePath)
if os.IsNotExist(err) {
info, err := v.folderFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
if _, err := os.Stat(v.versionsPath); err != nil {
if os.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsPath)
osutil.MkdirAll(v.versionsPath, 0755)
osutil.HideFile(v.versionsPath)
if _, err := v.versionsFs.Stat("."); err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsFs)
v.versionsFs.MkdirAll(".", 0755)
v.versionsFs.Hide(".")
} else {
return err
}
@@ -268,36 +261,41 @@ func (v *Staggered) Archive(filePath string) error {
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
inFolderPath := filepath.Dir(filePath)
if err != nil {
return err
}
dir := filepath.Join(v.versionsPath, inFolderPath)
err = osutil.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
err = v.versionsFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := taggedFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(dir, ver)
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(filePath, dst)
/// TODO: Fix this when we have an alternative filesystem implementation
if v.versionsFs.Type() != fs.FilesystemTypeBasic {
panic("bug: staggered versioner used with unsupported filesystem")
}
err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, taggedFilename(file, TimeGlob))
newVersions, err := osutil.Glob(pattern)
pattern := filepath.Join(inFolderPath, taggedFilename(file, TimeGlob))
newVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
}
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := osutil.Glob(pattern)
pattern = filepath.Join(inFolderPath, file+"~"+TimeGlob)
oldVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil