lib/model: Add fsync of files and directories, option to disable (fixes #3711)

gui/default/syncthing/core/syncthingController.js

@@ -1318,6 +1318,7 @@ angular.module('syncthing.core')
                 rescanIntervalS: 60,
                 minDiskFreePct: 1,
                 maxConflicts: 10,
+                fsync: true,
                 order: "random",
                 fileVersioningSelector: "none",
                 trashcanClean: 0,
@@ -1345,6 +1346,7 @@ angular.module('syncthing.core')
                 rescanIntervalS: 60,
                 minDiskFreePct: 1,
                 maxConflicts: 10,
+                fsync: true,
                 order: "random",
                 fileVersioningSelector: "none",
                 trashcanClean: 0,

lib/config/config.go

@@ -26,7 +26,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 16
+	CurrentVersion       = 17
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -254,6 +254,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 15 {
 		convertV15V16(cfg)
 	}
+	if cfg.Version == 16 {
+		convertV16V17(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -327,6 +330,14 @@ func convertV15V16(cfg *Configuration) {
 	cfg.Version = 16
 }
 
+func convertV16V17(cfg *Configuration) {
+	for i := range cfg.Folders {
+		cfg.Folders[i].Fsync = true
+	}
+
+	cfg.Version = 17
+}
+
 func convertV13V14(cfg *Configuration) {
 	// Not using the ignore cache is the new default. Disable it on existing
 	// configurations.

lib/config/config_test.go

@@ -104,6 +104,7 @@ func TestDeviceConfig(t *testing.T) {
 				AutoNormalize:   true,
 				MinDiskFreePct:  1,
 				MaxConflicts:    -1,
+				Fsync:           true,
 				Versioning: VersioningConfiguration{
 					Params: map[string]string{},
 				},

lib/config/folderconfiguration.go

@@ -38,6 +38,7 @@ type FolderConfiguration struct {
 	MaxConflicts          int                         `xml:"maxConflicts" json:"maxConflicts"`
 	DisableSparseFiles    bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
+	Fsync                 bool                        `xml:"fsync" json:"fsync"`
 
 	cachedPath string
 
@@ -85,6 +86,9 @@ func (f *FolderConfiguration) CreateMarker() error {
 			return err
 		}
 		fd.Close()
+		if err := osutil.SyncDir(filepath.Dir(marker)); err != nil {
+			l.Infof("fsync %q failed: %v", filepath.Dir(marker), err)
+		}
 		osutil.HideFile(marker)
 	}
 

lib/config/testdata/v17.xml

@@ -0,0 +1,15 @@
+<configuration version="17">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFreePct>1</minDiskFreePct>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/model/rwfolder.go

@@ -91,6 +91,7 @@ type rwFolder struct {
 	allowSparse    bool
 	checkFreeSpace bool
 	ignoreDelete   bool
+	fsync          bool
 
 	copiers int
 	pullers int
@@ -126,6 +127,7 @@ func newRWFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Ver
 		allowSparse:    !cfg.DisableSparseFiles,
 		checkFreeSpace: cfg.MinDiskFreePct != 0,
 		ignoreDelete:   cfg.IgnoreDelete,
+		fsync:          cfg.Fsync,
 
 		queue:       newJobQueue(),
 		pullTimer:   time.NewTimer(time.Second),
@@ -1372,12 +1374,50 @@ func (f *rwFolder) dbUpdaterRoutine() {
 	tick := time.NewTicker(maxBatchTime)
 	defer tick.Stop()
 
+	var changedFiles []string
+	var changedDirs []string
+	if f.fsync {
+		changedFiles = make([]string, 0, maxBatchSize)
+		changedDirs = make([]string, 0, maxBatchSize)
+	}
+
+	syncFilesOnce := func(files []string, syncFn func(string) error) {
+		sort.Strings(files)
+		var lastFile string
+		for _, file := range files {
+			if lastFile == file {
+				continue
+			}
+			lastFile = file
+			if err := syncFn(file); err != nil {
+				l.Infof("fsync %q failed: %v", file, err)
+			}
+		}
+	}
+
 	handleBatch := func() {
 		found := false
 		var lastFile protocol.FileInfo
 
 		for _, job := range batch {
 			files = append(files, job.file)
+			if f.fsync {
+				// collect changed files and dirs
+				switch job.jobType {
+				case dbUpdateHandleFile, dbUpdateShortcutFile:
+					// fsyncing symlinks is only supported by MacOS
+					if !job.file.IsSymlink() {
+						changedFiles = append(changedFiles,
+							filepath.Join(f.dir, job.file.Name))
+					}
+				case dbUpdateHandleDir:
+					changedDirs = append(changedDirs, filepath.Join(f.dir, job.file.Name))
+				}
+				if job.jobType != dbUpdateShortcutFile {
+					changedDirs = append(changedDirs,
+						filepath.Dir(filepath.Join(f.dir, job.file.Name)))
+				}
+			}
 			if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) {
 				continue
 			}
@@ -1390,6 +1430,14 @@ func (f *rwFolder) dbUpdaterRoutine() {
 			lastFile = job.file
 		}
 
+		if f.fsync {
+			// sync files and dirs to disk
+			syncFilesOnce(changedFiles, osutil.SyncFile)
+			changedFiles = changedFiles[:0]
+			syncFilesOnce(changedDirs, osutil.SyncDir)
+			changedDirs = changedDirs[:0]
+		}
+
 		// All updates to file/folder objects that originated remotely
 		// (across the network) use this call to updateLocals
 		f.model.updateLocalsFromPulling(f.folderID, files)

lib/osutil/atomic.go

@@ -77,6 +77,11 @@ func (w *AtomicWriter) Close() error {
 	// Try to not leave temp file around, but ignore error.
 	defer os.Remove(w.next.Name())
 
+	if err := w.next.Sync(); err != nil {
+		w.err = err
+		return err
+	}
+
 	if err := w.next.Close(); err != nil {
 		w.err = err
 		return err
@@ -97,6 +102,8 @@ func (w *AtomicWriter) Close() error {
 		return err
 	}
 
+	SyncDir(filepath.Dir(w.next.Name()))
+
 	// Set w.err to return appropriately for any future operations.
 	w.err = ErrClosed
 

lib/osutil/sync.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package osutil
+
+import (
+	"os"
+	"runtime"
+)
+
+func SyncFile(path string) error {
+	flag := 0
+	if runtime.GOOS == "windows" {
+		flag = os.O_WRONLY
+	}
+	fd, err := os.OpenFile(path, flag, 0)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+	// MacOS and Windows do not flush the disk cache
+	if err := fd.Sync(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func SyncDir(path string) error {
+	if runtime.GOOS == "windows" {
+		// not supported by Windows
+		return nil
+	}
+	return SyncFile(path)
+}