lib/config, lib/model: Configurable folder marker name (fixes #1126)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4483

lib/config/config.go

@@ -399,17 +399,21 @@ func convertV22V23(cfg *Configuration) {
 		// begin with.
 		permBits = 0700
 	}
+
+	// Upgrade code remains hardcoded for .stfolder despite configurable
+	// marker name in later versions.
+
 	for i := range cfg.Folders {
 		fs := cfg.Folders[i].Filesystem()
 		// Invalid config posted, or tests.
 		if fs == nil {
 			continue
 		}
-		if stat, err := fs.Stat(".stfolder"); err == nil && !stat.IsDir() {
+		if stat, err := fs.Stat(DefaultMarkerName); err == nil && !stat.IsDir() {
-			err = fs.Remove(".stfolder")
+			err = fs.Remove(DefaultMarkerName)
 			if err == nil {
-				err = fs.Mkdir(".stfolder", permBits)
+				err = fs.Mkdir(DefaultMarkerName, permBits)
-				fs.Hide(".stfolder") // ignore error
+				fs.Hide(DefaultMarkerName) // ignore error
 			}
 			if err != nil {
 				l.Infoln("Failed to upgrade folder marker:", err)

lib/config/config_test.go

@@ -85,13 +85,13 @@ func TestDefaultValues(t *testing.T) {
 
 func TestDeviceConfig(t *testing.T) {
 	for i := OldestHandledVersion; i <= CurrentVersion; i++ {
-		os.RemoveAll("testdata/.stfolder")
+		os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
 		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		_, err = os.Stat("testdata/.stfolder")
+		_, err = os.Stat(filepath.Join("testdata", DefaultMarkerName))
 		if i < 6 && err != nil {
 			t.Fatal(err)
 		} else if i >= 6 && err == nil {
@@ -120,6 +120,7 @@ func TestDeviceConfig(t *testing.T) {
 					Params: map[string]string{},
 				},
 				WeakHashThresholdPct: 25,
+				MarkerName:           DefaultMarkerName,
 			},
 		}
 

lib/config/folderconfiguration.go

@@ -20,6 +20,8 @@ var (
 	errMarkerMissing = errors.New("folder marker missing")
 )
 
+const DefaultMarkerName = ".stfolder"
+
 type FolderConfiguration struct {
 	ID                    string                      `xml:"id,attr" json:"id"`
 	Label                 string                      `xml:"label,attr" json:"label"`
@@ -47,6 +49,7 @@ type FolderConfiguration struct {
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
 	Paused                bool                        `xml:"paused" json:"paused"`
 	WeakHashThresholdPct  int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
+	MarkerName            string                      `xml:"markerName" json:"markerName"`
 
 	cachedFilesystem fs.Filesystem
 
@@ -91,6 +94,12 @@ func (f *FolderConfiguration) CreateMarker() error {
 	if err := f.CheckPath(); err != errMarkerMissing {
 		return err
 	}
+	if f.MarkerName != DefaultMarkerName {
+		// Folder uses a non-default marker so we shouldn't mess with it.
+		// Pretend we created it and let the subsequent health checks sort
+		// out the actual situation.
+		return nil
+	}
 
 	permBits := fs.FileMode(0777)
 	if runtime.GOOS == "windows" {
@@ -99,7 +108,7 @@ func (f *FolderConfiguration) CreateMarker() error {
 		permBits = 0700
 	}
 	fs := f.Filesystem()
-	err := fs.Mkdir(".stfolder", permBits)
+	err := fs.Mkdir(DefaultMarkerName, permBits)
 	if err != nil {
 		return err
 	}
@@ -108,7 +117,7 @@ func (f *FolderConfiguration) CreateMarker() error {
 	} else if err := dir.Sync(); err != nil {
 		l.Debugln("folder marker: fsync . failed:", err)
 	}
-	fs.Hide(".stfolder")
+	fs.Hide(DefaultMarkerName)
 
 	return nil
 }
@@ -120,7 +129,7 @@ func (f *FolderConfiguration) CheckPath() error {
 		return errPathMissing
 	}
 
-	_, err = f.Filesystem().Stat(".stfolder")
+	_, err = f.Filesystem().Stat(f.MarkerName)
 	if err != nil {
 		return errMarkerMissing
 	}
@@ -187,6 +196,10 @@ func (f *FolderConfiguration) prepare() {
 	if f.WeakHashThresholdPct == 0 {
 		f.WeakHashThresholdPct = 25
 	}
+
+	if f.MarkerName == "" {
+		f.MarkerName = DefaultMarkerName
+	}
 }
 
 type FolderDeviceConfigurationList []FolderDeviceConfiguration

lib/fs/filesystem.go

@@ -177,6 +177,7 @@ func NewFilesystem(fsType FilesystemType, uri string) Filesystem {
 // root, represents an internal file that should always be ignored. The file
 // path must be clean (i.e., in canonical shortest form).
 func IsInternal(file string) bool {
+	// fs cannot import config, so we hard code .stfolder here (config.DefaultMarkerName)
 	internals := []string{".stfolder", ".stignore", ".stversions"}
 	pathSep := string(PathSeparator)
 	for _, internal := range internals {

lib/model/model.go

@@ -254,7 +254,7 @@ func (m *Model) startFolderLocked(folder string) config.FolderType {
 	ffs := fs.MtimeFS()
 
 	// These are our metadata files, and they should always be hidden.
-	ffs.Hide(".stfolder")
+	ffs.Hide(config.DefaultMarkerName)
 	ffs.Hide(".stversions")
 	ffs.Hide(".stignore")
 
@@ -339,7 +339,7 @@ func (m *Model) RemoveFolder(cfg config.FolderConfiguration) {
 	m.fmut.Lock()
 	m.pmut.Lock()
 	// Delete syncthing specific files
-	cfg.Filesystem().RemoveAll(".stfolder")
+	cfg.Filesystem().RemoveAll(config.DefaultMarkerName)
 
 	m.tearDownFolderLocked(cfg.ID)
 	// Remove it from the database

lib/model/model_test.go

@@ -1029,8 +1029,8 @@ func changeIgnores(t *testing.T, m *Model, expected []string) {
 
 func TestIgnores(t *testing.T) {
 	// Assure a clean start state
-	os.RemoveAll("testdata/.stfolder")
+	os.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
-	os.MkdirAll("testdata/.stfolder", 0644)
+	os.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
 	ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
 
 	db := db.OpenMemory()
@@ -1106,6 +1106,7 @@ func TestROScanRecovery(t *testing.T) {
 		Path:            "testdata/rotestfolder",
 		Type:            config.FolderTypeSendOnly,
 		RescanIntervalS: 1,
+		MarkerName:      config.DefaultMarkerName,
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
@@ -1154,7 +1155,7 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.Path, config.DefaultMarkerName))
 	if err != nil {
 		t.Error(err)
 		return
@@ -1166,7 +1167,7 @@ func TestROScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.Path, config.DefaultMarkerName))
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
@@ -1193,6 +1194,7 @@ func TestRWScanRecovery(t *testing.T) {
 		Path:            "testdata/rwtestfolder",
 		Type:            config.FolderTypeSendReceive,
 		RescanIntervalS: 1,
+		MarkerName:      config.DefaultMarkerName,
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
@@ -1241,7 +1243,7 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	fd, err := os.Create(filepath.Join(fcfg.Path, ".stfolder"))
+	fd, err := os.Create(filepath.Join(fcfg.Path, config.DefaultMarkerName))
 	if err != nil {
 		t.Error(err)
 		return
@@ -1253,7 +1255,7 @@ func TestRWScanRecovery(t *testing.T) {
 		return
 	}
 
-	os.Remove(filepath.Join(fcfg.Path, ".stfolder"))
+	os.Remove(filepath.Join(fcfg.Path, config.DefaultMarkerName))
 
 	if err := waitFor("folder marker missing"); err != nil {
 		t.Error(err)
@@ -1760,16 +1762,16 @@ func TestUnifySubs(t *testing.T) {
 		{
 			// 6. .stignore and .stfolder are special and are passed on
 			// verbatim even though they are unknown
-			[]string{".stfolder", ".stignore"},
+			[]string{config.DefaultMarkerName, ".stignore"},
 			[]string{},
-			[]string{".stfolder", ".stignore"},
+			[]string{config.DefaultMarkerName, ".stignore"},
 		},
 		{
 			// 7. but the presence of something else unknown forces an actual
 			// scan
-			[]string{".stfolder", ".stignore", "foo/bar"},
+			[]string{config.DefaultMarkerName, ".stignore", "foo/bar"},
 			[]string{},
-			[]string{".stfolder", ".stignore", "foo"},
+			[]string{config.DefaultMarkerName, ".stignore", "foo"},
 		},
 		{
 			// 8. explicit request to scan all
@@ -2431,6 +2433,75 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	}
 }
 
+func TestCustomMarkerName(t *testing.T) {
+	ldb := db.OpenMemory()
+	set := db.NewFileSet("default", defaultFs, ldb)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	fcfg := config.FolderConfiguration{
+		ID:              "default",
+		Path:            "testdata/rwtestfolder",
+		Type:            config.FolderTypeSendReceive,
+		RescanIntervalS: 1,
+		MarkerName:      "myfile",
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+		Devices: []config.DeviceConfiguration{
+			{
+				DeviceID: device1,
+			},
+		},
+	})
+
+	os.RemoveAll(fcfg.Path)
+	defer os.RemoveAll(fcfg.Path)
+
+	m := NewModel(cfg, protocol.LocalDeviceID, "syncthing", "dev", ldb, nil)
+	m.AddFolder(fcfg)
+	m.StartFolder("default")
+	m.ServeBackground()
+	defer m.Stop()
+
+	waitFor := func(status string) error {
+		timeout := time.Now().Add(2 * time.Second)
+		for {
+			_, _, err := m.State("default")
+			if err == nil && status == "" {
+				return nil
+			}
+			if err != nil && err.Error() == status {
+				return nil
+			}
+
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err)
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}
+
+	if err := waitFor("folder path missing"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	os.Mkdir(fcfg.Path, 0700)
+	fd, err := os.Create(filepath.Join(fcfg.Path, "myfile"))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	fd.Close()
+
+	if err := waitFor(""); err != nil {
+		t.Error(err)
+		return
+	}
+}
+
 func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection {
 	fc := &fakeConnection{id: dev, model: m}
 	m.AddConnection(fc, protocol.HelloResult{})