perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Convert folders to use filesystem abstraction

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4228
This commit is contained in:
Audrius Butkevicius
2017-08-19 14:36:56 +00:00
committed by Jakob Borg
parent ab8c2fb5c7
commit 3d8b4a42b7
78 changed files with 2588 additions and 1665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
lib/config/config.go
View File

@@ -23,6 +23,7 @@ import (
"strconv"
"strings"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/upgrade"
@@ -31,7 +32,7 @@ import (
const (
OldestHandledVersion = 10
CurrentVersion = 21
CurrentVersion = 22
MaxRescanIntervalS = 365 * 24 * 60 * 60
)
@@ -319,6 +320,9 @@ func (cfg *Configuration) clean() error {
if cfg.Version == 20 {
convertV20V21(cfg)
}
if cfg.Version == 21 {
convertV21V22(cfg)
}
// Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool)
@@ -368,23 +372,38 @@ func (cfg *Configuration) clean() error {
return nil
}
func convertV21V22(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].FilesystemType = fs.FilesystemTypeBasic
// Migrate to templated external versioner commands
if cfg.Folders[i].Versioning.Type == "external" {
cfg.Folders[i].Versioning.Params["command"] += " %FOLDER_PATH% %FILE_PATH%"
}
}
cfg.Version = 22
}
func convertV20V21(cfg *Configuration) {
for _, folder := range cfg.Folders {
if folder.FilesystemType != fs.FilesystemTypeBasic {
continue
}
switch folder.Versioning.Type {
case "simple", "trashcan":
// Clean out symlinks in the known place
cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
cleanSymlinks(folder.Filesystem(), ".stversions")
case "staggered":
versionDir := folder.Versioning.Params["versionsPath"]
if versionDir == "" {
// default place
cleanSymlinks(filepath.Join(folder.Path(), ".stversions"))
cleanSymlinks(folder.Filesystem(), ".stversions")
} else if filepath.IsAbs(versionDir) {
// absolute
cleanSymlinks(versionDir)
cleanSymlinks(fs.NewFilesystem(fs.FilesystemTypeBasic, versionDir), ".")
} else {
// relative to folder
cleanSymlinks(filepath.Join(folder.Path(), versionDir))
cleanSymlinks(folder.Filesystem(), versionDir)
}
}
}
@@ -428,9 +447,7 @@ func convertV17V18(cfg *Configuration) {
}
func convertV16V17(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].Fsync = true
}
// Fsync = true removed
cfg.Version = 17
}
@@ -670,21 +687,21 @@ loop:
return devices[0:count]
}
func cleanSymlinks(dir string) {
func cleanSymlinks(filesystem fs.Filesystem, dir string) {
if runtime.GOOS == "windows" {
// We don't do symlinks on Windows. Additionally, there may
// be things that look like symlinks that are not, which we
// should leave alone. Deduplicated files, for example.
return
}
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
filesystem.Walk(dir, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if info.IsSymlink() {
l.Infoln("Removing incorrectly versioned symlink", path)
os.Remove(path)
return filepath.SkipDir
filesystem.Remove(path)
return fs.SkipDir
}
return nil
})

63
lib/config/config_test.go
View File

@@ -19,6 +19,7 @@ import (
"testing"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -103,7 +104,8 @@ func TestDeviceConfig(t *testing.T) {
expectedFolders := []FolderConfiguration{
{
ID: "test",
RawPath: "testdata",
FilesystemType: fs.FilesystemTypeBasic,
Path: "testdata",
Devices: []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
Type: FolderTypeSendOnly,
RescanIntervalS: 600,
@@ -113,7 +115,6 @@ func TestDeviceConfig(t *testing.T) {
AutoNormalize: true,
MinDiskFree: Size{1, "%"},
MaxConflicts: -1,
Fsync: true,
Versioning: VersioningConfiguration{
Params: map[string]string{},
},
@@ -121,15 +122,11 @@ func TestDeviceConfig(t *testing.T) {
},
}
// The cachedPath will have been resolved to an absolute path,
// The cachedFilesystem will have been resolved to an absolute path,
// depending on where the tests are running. Zero it out so we don't
// fail based on that.
for i := range cfg.Folders {
cfg.Folders[i].cachedPath = ""
}
if runtime.GOOS != "windows" {
expectedFolders[0].RawPath += string(filepath.Separator)
cfg.Folders[i].cachedFilesystem = nil
}
expectedDevices := []DeviceConfiguration{
@@ -377,16 +374,17 @@ func TestVersioningConfig(t *testing.T) {
}
func TestIssue1262(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
}
cfg, err := Load("testdata/issue-1262.xml", device4)
if err != nil {
t.Fatal(err)
}
actual := cfg.Folders()["test"].RawPath
expected := "e:/"
if runtime.GOOS == "windows" {
expected = `e:\`
}
actual := cfg.Folders()["test"].Filesystem().URI()
expected := `e:\`
if actual != expected {
t.Errorf("%q != %q", actual, expected)
@@ -416,43 +414,12 @@ func TestIssue1750(t *testing.T) {
}
}
func TestWindowsPaths(t *testing.T) {
if runtime.GOOS != "windows" {
t.Skip("Not useful on non-Windows")
return
}
folder := FolderConfiguration{
RawPath: `e:\`,
}
expected := `\\?\e:\`
actual := folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `\\192.0.2.22\network\share`
expected = folder.RawPath
actual = folder.Path()
if actual != expected {
t.Errorf("%q != %q", actual, expected)
}
folder.RawPath = `relative\path`
expected = folder.RawPath
actual = folder.Path()
if actual == expected || !strings.HasPrefix(actual, "\\\\?\\") {
t.Errorf("%q == %q, expected absolutification", actual, expected)
}
}
func TestFolderPath(t *testing.T) {
folder := FolderConfiguration{
RawPath: "~/tmp",
Path: "~/tmp",
}
realPath := folder.Path()
realPath := folder.Filesystem().URI()
if !filepath.IsAbs(realPath) {
t.Error(realPath, "should be absolute")
}
@@ -677,8 +644,8 @@ func TestEmptyFolderPaths(t *testing.T) {
t.Fatal(err)
}
folder := wrapper.Folders()["f1"]
if folder.Path() != "" {
t.Errorf("Expected %q to be empty", folder.Path())
if folder.cachedFilesystem != nil {
t.Errorf("Expected %q to be empty", folder.cachedFilesystem)
}
}

113
lib/config/folderconfiguration.go
View File

@@ -8,19 +8,17 @@ package config
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
type FolderConfiguration struct {
ID string `xml:"id,attr" json:"id"`
Label string `xml:"label,attr" json:"label"`
RawPath string `xml:"path,attr" json:"path"`
FilesystemType fs.FilesystemType `xml:"filesystemType" json:"filesystemType"`
Path string `xml:"path,attr" json:"path"`
Type FolderType `xml:"type,attr" json:"type"`
Devices []FolderDeviceConfiguration `xml:"device" json:"devices"`
RescanIntervalS int `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
@@ -39,11 +37,10 @@ type FolderConfiguration struct {
MaxConflicts int `xml:"maxConflicts" json:"maxConflicts"`
DisableSparseFiles bool `xml:"disableSparseFiles" json:"disableSparseFiles"`
DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"`
Fsync bool `xml:"fsync" json:"fsync"`
Paused bool `xml:"paused" json:"paused"`
WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
cachedPath string
cachedFilesystem fs.Filesystem
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@@ -54,10 +51,11 @@ type FolderDeviceConfiguration struct {
IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"`
}
func NewFolderConfiguration(id, path string) FolderConfiguration {
func NewFolderConfiguration(id string, fsType fs.FilesystemType, path string) FolderConfiguration {
f := FolderConfiguration{
ID: id,
RawPath: path,
ID: id,
FilesystemType: fsType,
Path: path,
}
f.prepare()
return f
@@ -71,53 +69,57 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
return c
}
func (f FolderConfiguration) Path() string {
func (f FolderConfiguration) Filesystem() fs.Filesystem {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Path() should be valid.
if f.cachedPath == "" && f.RawPath != "" {
l.Infoln("bug: uncached path call (should only happen in tests)")
return f.cleanedPath()
// cfg.Folders["default"].Filesystem() should be valid.
if f.cachedFilesystem == nil && f.Path != "" {
l.Infoln("bug: uncached filesystem call (should only happen in tests)")
return fs.NewFilesystem(f.FilesystemType, f.Path)
}
return f.cachedPath
return f.cachedFilesystem
}
func (f *FolderConfiguration) CreateMarker() error {
if !f.HasMarker() {
marker := filepath.Join(f.Path(), ".stfolder")
fd, err := os.Create(marker)
fs := f.Filesystem()
fd, err := fs.Create(".stfolder")
if err != nil {
return err
}
fd.Close()
if err := osutil.SyncDir(filepath.Dir(marker)); err != nil {
l.Infof("fsync %q failed: %v", filepath.Dir(marker), err)
if dir, err := fs.Open("."); err == nil {
if serr := dir.Sync(); err != nil {
l.Infof("fsync %q failed: %v", ".", serr)
}
} else {
l.Infof("fsync %q failed: %v", ".", err)
}
osutil.HideFile(marker)
fs.Hide(".stfolder")
}
return nil
}
func (f *FolderConfiguration) HasMarker() bool {
_, err := os.Stat(filepath.Join(f.Path(), ".stfolder"))
_, err := f.Filesystem().Stat(".stfolder")
return err == nil
}
func (f *FolderConfiguration) CreateRoot() (err error) {
// Directory permission bits. Will be filtered down to something
// sane by umask on Unixes.
permBits := os.FileMode(0777)
permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" {
// Windows has no umask so we must chose a safer set of bits to
// begin with.
permBits = 0700
}
if _, err = os.Stat(f.Path()); os.IsNotExist(err) {
if err = osutil.MkdirAll(f.Path(), permBits); err != nil {
l.Warnf("Creating directory for %v: %v",
f.Description(), err)
filesystem := f.Filesystem()
if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
if err = filesystem.MkdirAll(".", permBits); err != nil {
l.Warnf("Creating directory for %v: %v", f.Description(), err)
}
}
@@ -140,24 +142,10 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
}
func (f *FolderConfiguration) prepare() {
if f.RawPath != "" {
// The reason it's done like this:
// C: -> C:\ -> C:\ (issue that this is trying to fix)
// C:\somedir -> C:\somedir\ -> C:\somedir
// C:\somedir\ -> C:\somedir\\ -> C:\somedir
// This way in the tests, we get away without OS specific separators
// in the test configs.
f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
// If we're not on Windows, we want the path to end with a slash to
// penetrate symlinks. On Windows, paths must not end with a slash.
if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
f.RawPath = f.RawPath + string(filepath.Separator)
}
if f.Path != "" {
f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
}
f.cachedPath = f.cleanedPath()
if f.RescanIntervalS > MaxRescanIntervalS {
f.RescanIntervalS = MaxRescanIntervalS
} else if f.RescanIntervalS < 0 {
@@ -173,43 +161,6 @@ func (f *FolderConfiguration) prepare() {
}
}
func (f *FolderConfiguration) cleanedPath() string {
if f.RawPath == "" {
return ""
}
cleaned := f.RawPath
// Attempt tilde expansion; leave unchanged in case of error
if path, err := osutil.ExpandTilde(cleaned); err == nil {
cleaned = path
}
// Attempt absolutification; leave unchanged in case of error
if !filepath.IsAbs(cleaned) {
// Abs() looks like a fairly expensive syscall on Windows, while
// IsAbs() is a whole bunch of string mangling. I think IsAbs() may be
// somewhat faster in the general case, hence the outer if...
if path, err := filepath.Abs(cleaned); err == nil {
cleaned = path
}
}
// Attempt to enable long filename support on Windows. We may still not
// have an absolute path here if the previous steps failed.
if runtime.GOOS == "windows" && filepath.IsAbs(cleaned) && !strings.HasPrefix(f.RawPath, `\\`) {
return `\\?\` + cleaned
}
// If we're not on Windows, we want the path to end with a slash to
// penetrate symlinks. On Windows, paths must not end with a slash.
if runtime.GOOS != "windows" && cleaned[len(cleaned)-1] != filepath.Separator {
cleaned = cleaned + string(filepath.Separator)
}
return cleaned
}
type FolderDeviceConfigurationList []FolderDeviceConfiguration
func (l FolderDeviceConfigurationList) Less(a, b int) bool {

16
lib/config/testdata/v22.xml vendored Normal file
View File

@@ -0,0 +1,16 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

4
lib/config/testdata/versioningconfig.xml vendored
View File

@@ -1,5 +1,5 @@
<configuration version="10">
<folder id="test" directory="testdata/" ro="true">
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<versioning type="simple">
<param key="foo" val="bar"/>
<param key="baz" val="quux"/>