From 9c855ab22e9f0e895db44fd63e0460c0c9451f6f Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 5 Nov 2017 12:18:05 +0000 Subject: [PATCH] lib/config, lib/model: Configurable folder marker name (fixes #1126) GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4483 --- lib/config/config.go | 12 ++-- lib/config/config_test.go | 5 +- lib/config/folderconfiguration.go | 19 ++++++- lib/fs/filesystem.go | 1 + lib/model/model.go | 4 +- lib/model/model_test.go | 91 +++++++++++++++++++++++++++---- 6 files changed, 111 insertions(+), 21 deletions(-) diff --git a/lib/config/config.go b/lib/config/config.go index 61ceb3ec..bf820c07 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -399,17 +399,21 @@ func convertV22V23(cfg *Configuration) { // begin with. permBits = 0700 } + + // Upgrade code remains hardcoded for .stfolder despite configurable + // marker name in later versions. + for i := range cfg.Folders { fs := cfg.Folders[i].Filesystem() // Invalid config posted, or tests. if fs == nil { continue } - if stat, err := fs.Stat(".stfolder"); err == nil && !stat.IsDir() { - err = fs.Remove(".stfolder") + if stat, err := fs.Stat(DefaultMarkerName); err == nil && !stat.IsDir() { + err = fs.Remove(DefaultMarkerName) if err == nil { - err = fs.Mkdir(".stfolder", permBits) - fs.Hide(".stfolder") // ignore error + err = fs.Mkdir(DefaultMarkerName, permBits) + fs.Hide(DefaultMarkerName) // ignore error } if err != nil { l.Infoln("Failed to upgrade folder marker:", err) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 2cd5b533..ba03db5a 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -85,13 +85,13 @@ func TestDefaultValues(t *testing.T) { func TestDeviceConfig(t *testing.T) { for i := OldestHandledVersion; i <= CurrentVersion; i++ { - os.RemoveAll("testdata/.stfolder") + os.RemoveAll(filepath.Join("testdata", DefaultMarkerName)) wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1) if err != nil { t.Fatal(err) } - _, err = os.Stat("testdata/.stfolder") + _, err = os.Stat(filepath.Join("testdata", DefaultMarkerName)) if i < 6 && err != nil { t.Fatal(err) } else if i >= 6 && err == nil { @@ -120,6 +120,7 @@ func TestDeviceConfig(t *testing.T) { Params: map[string]string{}, }, WeakHashThresholdPct: 25, + MarkerName: DefaultMarkerName, }, } diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index 676d82b5..18fb1c6a 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -20,6 +20,8 @@ var ( errMarkerMissing = errors.New("folder marker missing") ) +const DefaultMarkerName = ".stfolder" + type FolderConfiguration struct { ID string `xml:"id,attr" json:"id"` Label string `xml:"label,attr" json:"label"` @@ -47,6 +49,7 @@ type FolderConfiguration struct { DisableTempIndexes bool `xml:"disableTempIndexes" json:"disableTempIndexes"` Paused bool `xml:"paused" json:"paused"` WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash. + MarkerName string `xml:"markerName" json:"markerName"` cachedFilesystem fs.Filesystem @@ -91,6 +94,12 @@ func (f *FolderConfiguration) CreateMarker() error { if err := f.CheckPath(); err != errMarkerMissing { return err } + if f.MarkerName != DefaultMarkerName { + // Folder uses a non-default marker so we shouldn't mess with it. + // Pretend we created it and let the subsequent health checks sort + // out the actual situation. + return nil + } permBits := fs.FileMode(0777) if runtime.GOOS == "windows" { @@ -99,7 +108,7 @@ func (f *FolderConfiguration) CreateMarker() error { permBits = 0700 } fs := f.Filesystem() - err := fs.Mkdir(".stfolder", permBits) + err := fs.Mkdir(DefaultMarkerName, permBits) if err != nil { return err } @@ -108,7 +117,7 @@ func (f *FolderConfiguration) CreateMarker() error { } else if err := dir.Sync(); err != nil { l.Debugln("folder marker: fsync . failed:", err) } - fs.Hide(".stfolder") + fs.Hide(DefaultMarkerName) return nil } @@ -120,7 +129,7 @@ func (f *FolderConfiguration) CheckPath() error { return errPathMissing } - _, err = f.Filesystem().Stat(".stfolder") + _, err = f.Filesystem().Stat(f.MarkerName) if err != nil { return errMarkerMissing } @@ -187,6 +196,10 @@ func (f *FolderConfiguration) prepare() { if f.WeakHashThresholdPct == 0 { f.WeakHashThresholdPct = 25 } + + if f.MarkerName == "" { + f.MarkerName = DefaultMarkerName + } } type FolderDeviceConfigurationList []FolderDeviceConfiguration diff --git a/lib/fs/filesystem.go b/lib/fs/filesystem.go index f7f27e7d..fe267ac0 100644 --- a/lib/fs/filesystem.go +++ b/lib/fs/filesystem.go @@ -177,6 +177,7 @@ func NewFilesystem(fsType FilesystemType, uri string) Filesystem { // root, represents an internal file that should always be ignored. The file // path must be clean (i.e., in canonical shortest form). func IsInternal(file string) bool { + // fs cannot import config, so we hard code .stfolder here (config.DefaultMarkerName) internals := []string{".stfolder", ".stignore", ".stversions"} pathSep := string(PathSeparator) for _, internal := range internals { diff --git a/lib/model/model.go b/lib/model/model.go index 62aa87c2..9c9b1c6a 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -254,7 +254,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType { ffs := fs.MtimeFS() // These are our metadata files, and they should always be hidden. - ffs.Hide(".stfolder") + ffs.Hide(config.DefaultMarkerName) ffs.Hide(".stversions") ffs.Hide(".stignore") @@ -339,7 +339,7 @@ func (m *Model) RemoveFolder(cfg config.FolderConfiguration) { m.fmut.Lock() m.pmut.Lock() // Delete syncthing specific files - cfg.Filesystem().RemoveAll(".stfolder") + cfg.Filesystem().RemoveAll(config.DefaultMarkerName) m.tearDownFolderLocked(cfg.ID) // Remove it from the database diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 4d2242b1..6304e6f8 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -1029,8 +1029,8 @@ func changeIgnores(t *testing.T, m *Model, expected []string) { func TestIgnores(t *testing.T) { // Assure a clean start state - os.RemoveAll("testdata/.stfolder") - os.MkdirAll("testdata/.stfolder", 0644) + os.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName)) + os.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644) ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644) db := db.OpenMemory() @@ -1106,6 +1106,7 @@ func TestROScanRecovery(t *testing.T) { Path: "testdata/rotestfolder", Type: config.FolderTypeSendOnly, RescanIntervalS: 1, + MarkerName: config.DefaultMarkerName, } cfg := config.Wrap("/tmp/test", config.Configuration{ Folders: []config.FolderConfiguration{fcfg}, @@ -1154,7 +1155,7 @@ func TestROScanRecovery(t *testing.T) { return } - fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) + fd, err := os.Create(filepath.Join(fcfg.Path, config.DefaultMarkerName)) if err != nil { t.Error(err) return @@ -1166,7 +1167,7 @@ func TestROScanRecovery(t *testing.T) { return } - os.Remove(filepath.Join(fcfg.Path, ".stfolder")) + os.Remove(filepath.Join(fcfg.Path, config.DefaultMarkerName)) if err := waitFor("folder marker missing"); err != nil { t.Error(err) @@ -1193,6 +1194,7 @@ func TestRWScanRecovery(t *testing.T) { Path: "testdata/rwtestfolder", Type: config.FolderTypeSendReceive, RescanIntervalS: 1, + MarkerName: config.DefaultMarkerName, } cfg := config.Wrap("/tmp/test", config.Configuration{ Folders: []config.FolderConfiguration{fcfg}, @@ -1241,7 +1243,7 @@ func TestRWScanRecovery(t *testing.T) { return } - fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder")) + fd, err := os.Create(filepath.Join(fcfg.Path, config.DefaultMarkerName)) if err != nil { t.Error(err) return @@ -1253,7 +1255,7 @@ func TestRWScanRecovery(t *testing.T) { return } - os.Remove(filepath.Join(fcfg.Path, ".stfolder")) + os.Remove(filepath.Join(fcfg.Path, config.DefaultMarkerName)) if err := waitFor("folder marker missing"); err != nil { t.Error(err) @@ -1760,16 +1762,16 @@ func TestUnifySubs(t *testing.T) { { // 6. .stignore and .stfolder are special and are passed on // verbatim even though they are unknown - []string{".stfolder", ".stignore"}, + []string{config.DefaultMarkerName, ".stignore"}, []string{}, - []string{".stfolder", ".stignore"}, + []string{config.DefaultMarkerName, ".stignore"}, }, { // 7. but the presence of something else unknown forces an actual // scan - []string{".stfolder", ".stignore", "foo/bar"}, + []string{config.DefaultMarkerName, ".stignore", "foo/bar"}, []string{}, - []string{".stfolder", ".stignore", "foo"}, + []string{config.DefaultMarkerName, ".stignore", "foo"}, }, { // 8. explicit request to scan all @@ -2431,6 +2433,75 @@ func TestNoRequestsFromPausedDevices(t *testing.T) { } } +func TestCustomMarkerName(t *testing.T) { + ldb := db.OpenMemory() + set := db.NewFileSet("default", defaultFs, ldb) + set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ + {Name: "dummyfile"}, + }) + + fcfg := config.FolderConfiguration{ + ID: "default", + Path: "testdata/rwtestfolder", + Type: config.FolderTypeSendReceive, + RescanIntervalS: 1, + MarkerName: "myfile", + } + cfg := config.Wrap("/tmp/test", config.Configuration{ + Folders: []config.FolderConfiguration{fcfg}, + Devices: []config.DeviceConfiguration{ + { + DeviceID: device1, + }, + }, + }) + + os.RemoveAll(fcfg.Path) + defer os.RemoveAll(fcfg.Path) + + m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil) + m.AddFolder(fcfg) + m.StartFolder("default") + m.ServeBackground() + defer m.Stop() + + waitFor := func(status string) error { + timeout := time.Now().Add(2 * time.Second) + for { + _, _, err := m.State("default") + if err == nil && status == "" { + return nil + } + if err != nil && err.Error() == status { + return nil + } + + if time.Now().After(timeout) { + return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err) + } + time.Sleep(10 * time.Millisecond) + } + } + + if err := waitFor("folder path missing"); err != nil { + t.Error(err) + return + } + + os.Mkdir(fcfg.Path, 0700) + fd, err := os.Create(filepath.Join(fcfg.Path, "myfile")) + if err != nil { + t.Error(err) + return + } + fd.Close() + + if err := waitFor(""); err != nil { + t.Error(err) + return + } +} + func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection { fc := &fakeConnection{id: dev, model: m} m.AddConnection(fc, protocol.HelloResult{})