all: Copy owner/group from parent (fixes #5445) (#5479)

This adds a folder option "CopyOwnershipFromParent" which, when set,
makes Syncthing attempt to retain the owner/group information when
syncing files. Specifically, at the finisher stage we look at the parent
dir to get owner/group and then attempt a Lchown call on the temp file.
For this to succeed Syncthing must be running with the appropriate
permissions. On Linux this is CAP_FOWNER, which can be granted by the
service manager on startup or set on the binary in the filesystem. Other
operating systems do other things, but often it's not required to run as
full "root". On Windows this patch does nothing - ownership works
differently there and is generally less of a deal, as permissions are
inherited as ACLs anyway.

There are unit tests on the Lchown functionality, which requires the
above permissions to run. There is also a unit test on the folder which
uses the fake filesystem and hence does not need special permissions.

lib/config/folderconfiguration.go

@@ -53,6 +53,7 @@ type FolderConfiguration struct {
 	WeakHashThresholdPct    int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
 	MarkerName              string                      `xml:"markerName" json:"markerName"`
 	UseLargeBlocks          bool                        `xml:"useLargeBlocks" json:"useLargeBlocks"`
+	CopyOwnershipFromParent bool                        `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
 
 	cachedFilesystem fs.Filesystem
 

lib/fs/basicfs.go

@@ -98,6 +98,14 @@ func (f *BasicFilesystem) Chmod(name string, mode FileMode) error {
 	return os.Chmod(name, os.FileMode(mode))
 }
 
+func (f *BasicFilesystem) Lchown(name string, uid, gid int) error {
+	name, err := f.rooted(name)
+	if err != nil {
+		return err
+	}
+	return os.Lchown(name, uid, gid)
+}
+
 func (f *BasicFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
 	name, err := f.rooted(name)
 	if err != nil {

lib/fs/basicfs_test.go

@@ -15,6 +15,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/rand"
 )
 
 func setup(t *testing.T) (*BasicFilesystem, string) {
@@ -56,6 +58,54 @@ func TestChmodFile(t *testing.T) {
 	}
 }
 
+func TestChownFile(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Not supported on Windows")
+		return
+	}
+	if os.Getuid() != 0 {
+		// We are not root. No expectation of being able to chown. Our tests
+		// typically don't run with CAP_FOWNER.
+		t.Skip("Test not possible")
+		return
+	}
+
+	fs, dir := setup(t)
+	path := filepath.Join(dir, "file")
+	defer os.RemoveAll(dir)
+
+	defer os.Chmod(path, 0666)
+
+	fd, err := os.Create(path)
+	if err != nil {
+		t.Error("Unexpected error:", err)
+	}
+	fd.Close()
+
+	info, err := fs.Lstat("file")
+	if err != nil {
+		t.Error("Unexpected error:", err)
+	}
+
+	newUID := 1000 + rand.Intn(30000)
+	newGID := 1000 + rand.Intn(30000)
+
+	if err := fs.Lchown("file", newUID, newGID); err != nil {
+		t.Error("Unexpected error:", err)
+	}
+
+	info, err = fs.Lstat("file")
+	if err != nil {
+		t.Error("Unexpected error:", err)
+	}
+	if info.Owner() != newUID {
+		t.Errorf("Incorrect owner, expected %d but got %d", newUID, info.Owner())
+	}
+	if info.Group() != newGID {
+		t.Errorf("Incorrect group, expected %d but got %d", newGID, info.Group())
+	}
+}
+
 func TestChmodDir(t *testing.T) {
 	fs, dir := setup(t)
 	path := filepath.Join(dir, "dir")

lib/fs/errorfs.go

@@ -18,6 +18,7 @@ type errorFilesystem struct {
 }
 
 func (fs *errorFilesystem) Chmod(name string, mode FileMode) error                      { return fs.err }
+func (fs *errorFilesystem) Lchown(name string, uid, gid int) error                      { return fs.err }
 func (fs *errorFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { return fs.err }
 func (fs *errorFilesystem) Create(name string) (File, error)                            { return nil, fs.err }
 func (fs *errorFilesystem) CreateSymlink(target, name string) error                     { return fs.err }

lib/fs/fakefs.go

@@ -79,7 +79,7 @@ func newFakeFilesystem(root string) *fakefs {
 	fs := &fakefs{
 		root: &fakeEntry{
 			name:      "/",
-			isdir:    true,
+			entryType: fakeEntryTypeDir,
 			mode:      0700,
 			mtime:     time.Now(),
 			children:  make(map[string]*fakeEntry),
@@ -126,17 +126,30 @@ func newFakeFilesystem(root string) *fakefs {
 	return fs
 }
 
+type fakeEntryType int
+
+const (
+	fakeEntryTypeFile fakeEntryType = iota
+	fakeEntryTypeDir
+	fakeEntryTypeSymlink
+)
+
 // fakeEntry is an entry (file or directory) in the fake filesystem
 type fakeEntry struct {
 	name      string
-	isdir    bool
+	entryType fakeEntryType
+	dest      string // for symlinks
 	size      int64
 	mode      FileMode
+	uid       int
+	gid       int
 	mtime     time.Time
 	children  map[string]*fakeEntry
 }
 
 func (fs *fakefs) entryForName(name string) *fakeEntry {
+	// bug: lookup doesn't work through symlinks.
+
 	name = filepath.ToSlash(name)
 	if name == "." || name == "/" {
 		return fs.root
@@ -146,6 +159,9 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
 	comps := strings.Split(name, "/")
 	entry := fs.root
 	for _, comp := range comps {
+		if entry.entryType != fakeEntryTypeDir {
+			return nil
+		}
 		var ok bool
 		entry, ok = entry.children[comp]
 		if !ok {
@@ -166,6 +182,18 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
 	return nil
 }
 
+func (fs *fakefs) Lchown(name string, uid, gid int) error {
+	fs.mut.Lock()
+	defer fs.mut.Unlock()
+	entry := fs.entryForName(name)
+	if entry == nil {
+		return os.ErrNotExist
+	}
+	entry.uid = uid
+	entry.gid = gid
+	return nil
+}
+
 func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
@@ -177,18 +205,20 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
 	return nil
 }
 
-func (fs *fakefs) Create(name string) (File, error) {
+func (fs *fakefs) create(name string) (*fakeEntry, error) {
 	fs.mut.Lock()
 	defer fs.mut.Unlock()
 
 	if entry := fs.entryForName(name); entry != nil {
-		if entry.isdir {
+		if entry.entryType == fakeEntryTypeDir {
 			return nil, os.ErrExist
+		} else if entry.entryType == fakeEntryTypeSymlink {
+			return nil, errors.New("following symlink not supported")
 		}
 		entry.size = 0
 		entry.mtime = time.Now()
 		entry.mode = 0666
-		return &fakeFile{fakeEntry: entry}, nil
+		return entry, nil
 	}
 
 	dir := filepath.Dir(name)
@@ -203,11 +233,25 @@ func (fs *fakefs) Create(name string) (File, error) {
 		mtime: time.Now(),
 	}
 	entry.children[base] = new
-	return &fakeFile{fakeEntry: new}, nil
+	return new, nil
+}
+
+func (fs *fakefs) Create(name string) (File, error) {
+	entry, err := fs.create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &fakeFile{fakeEntry: entry}, nil
 }
 
 func (fs *fakefs) CreateSymlink(target, name string) error {
-	return errors.New("not implemented")
+	entry, err := fs.create(name)
+	if err != nil {
+		return err
+	}
+	entry.entryType = fakeEntryTypeSymlink
+	entry.dest = target
+	return nil
 }
 
 func (fs *fakefs) DirNames(name string) ([]string, error) {
@@ -248,13 +292,16 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
 	if entry == nil {
 		return os.ErrNotExist
 	}
+	if entry.entryType != fakeEntryTypeDir {
+		return os.ErrExist
+	}
 	if _, ok := entry.children[base]; ok {
 		return os.ErrExist
 	}
 
 	entry.children[base] = &fakeEntry{
 		name:      base,
-		isdir:    true,
+		entryType: fakeEntryTypeDir,
 		mode:      perm,
 		mtime:     time.Now(),
 		children:  make(map[string]*fakeEntry),
@@ -273,14 +320,14 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
 		if !ok {
 			new := &fakeEntry{
 				name:      comp,
-				isdir:    true,
+				entryType: fakeEntryTypeDir,
 				mode:      perm,
 				mtime:     time.Now(),
 				children:  make(map[string]*fakeEntry),
 			}
 			entry.children[comp] = new
 			next = new
-		} else if !next.isdir {
+		} else if next.entryType != fakeEntryTypeDir {
 			return errors.New("not a directory")
 		}
 
@@ -294,7 +341,7 @@ func (fs *fakefs) Open(name string) (File, error) {
 	defer fs.mut.Unlock()
 
 	entry := fs.entryForName(name)
-	if entry == nil {
+	if entry == nil || entry.entryType != fakeEntryTypeFile {
 		return nil, os.ErrNotExist
 	}
 	return &fakeFile{fakeEntry: entry}, nil
@@ -313,6 +360,8 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 	entry := fs.entryForName(dir)
 	if entry == nil {
 		return nil, os.ErrNotExist
+	} else if entry.entryType != fakeEntryTypeDir {
+		return nil, errors.New("not a directory")
 	}
 
 	if flags&os.O_EXCL != 0 {
@@ -332,7 +381,16 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 }
 
 func (fs *fakefs) ReadSymlink(name string) (string, error) {
-	return "", errors.New("not implemented")
+	fs.mut.Lock()
+	defer fs.mut.Unlock()
+
+	entry := fs.entryForName(name)
+	if entry == nil {
+		return "", os.ErrNotExist
+	} else if entry.entryType != fakeEntryTypeSymlink {
+		return "", errors.New("not a symlink")
+	}
+	return entry.dest, nil
 }
 
 func (fs *fakefs) Remove(name string) error {
@@ -387,7 +445,7 @@ func (fs *fakefs) Rename(oldname, newname string) error {
 	}
 
 	dst, ok := p1.children[filepath.Base(newname)]
-	if ok && dst.isdir {
+	if ok && dst.entryType == fakeEntryTypeDir {
 		return errors.New("is a directory")
 	}
 
@@ -513,7 +571,7 @@ func (f *fakeFile) readShortAt(p []byte, offs int64) (int, error) {
 	// start of the block to serve a given read. 128 KiB blocks fit
 	// reasonably well with the type of IO Syncthing tends to do.
 
-	if f.isdir {
+	if f.entryType == fakeEntryTypeDir {
 		return 0, errors.New("is a directory")
 	}
 
@@ -570,7 +628,7 @@ func (f *fakeFile) Seek(offset int64, whence int) (int64, error) {
 	f.mut.Lock()
 	defer f.mut.Unlock()
 
-	if f.isdir {
+	if f.entryType == fakeEntryTypeDir {
 		return 0, errors.New("is a directory")
 	}
 
@@ -603,7 +661,7 @@ func (f *fakeFile) WriteAt(p []byte, off int64) (int, error) {
 	f.mut.Lock()
 	defer f.mut.Unlock()
 
-	if f.isdir {
+	if f.entryType == fakeEntryTypeDir {
 		return 0, errors.New("is a directory")
 	}
 
@@ -661,13 +719,21 @@ func (f *fakeFileInfo) ModTime() time.Time {
 }
 
 func (f *fakeFileInfo) IsDir() bool {
-	return f.isdir
+	return f.entryType == fakeEntryTypeDir
 }
 
 func (f *fakeFileInfo) IsRegular() bool {
-	return !f.isdir
+	return f.entryType == fakeEntryTypeFile
 }
 
 func (f *fakeFileInfo) IsSymlink() bool {
-	return false
+	return f.entryType == fakeEntryTypeSymlink
+}
+
+func (f *fakeFileInfo) Owner() int {
+	return f.uid
+}
+
+func (f *fakeFileInfo) Group() int {
+	return f.gid
 }

lib/fs/fakefs_test.go

@@ -101,6 +101,26 @@ func TestFakeFS(t *testing.T) {
 	if !bytes.Equal(bs0, bs1[1:]) {
 		t.Error("wrong data")
 	}
+
+	// Create symlink
+	if err := fs.CreateSymlink("foo", "dira/dirb/symlink"); err != nil {
+		t.Fatal(err)
+	}
+	if str, err := fs.ReadSymlink("dira/dirb/symlink"); err != nil {
+		t.Fatal(err)
+	} else if str != "foo" {
+		t.Error("Wrong symlink destination", str)
+	}
+
+	// Chown
+	if err := fs.Lchown("dira", 1234, 5678); err != nil {
+		t.Fatal(err)
+	}
+	if info, err := fs.Lstat("dira"); err != nil {
+		t.Fatal(err)
+	} else if info.Owner() != 1234 || info.Group() != 5678 {
+		t.Error("Wrong owner/group")
+	}
 }
 
 func TestFakeFSRead(t *testing.T) {

lib/fs/fileinfo_unix.go

@@ -0,0 +1,25 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+import "syscall"
+
+func (e fsFileInfo) Owner() int {
+	if st, ok := e.Sys().(*syscall.Stat_t); ok {
+		return int(st.Uid)
+	}
+	return -1
+}
+
+func (e fsFileInfo) Group() int {
+	if st, ok := e.Sys().(*syscall.Stat_t); ok {
+		return int(st.Gid)
+	}
+	return -1
+}

lib/fs/fileinfo_windows.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package fs
+
+func (e fsFileInfo) Owner() int {
+	return -1
+}
+
+func (e fsFileInfo) Group() int {
+	return -1
+}

lib/fs/filesystem.go

@@ -19,6 +19,7 @@ import (
 // The Filesystem interface abstracts access to the file system.
 type Filesystem interface {
 	Chmod(name string, mode FileMode) error
+	Lchown(name string, uid, gid int) error
 	Chtimes(name string, atime time.Time, mtime time.Time) error
 	Create(name string) (File, error)
 	CreateSymlink(target, name string) error
@@ -74,6 +75,8 @@ type FileInfo interface {
 	// Extensions
 	IsRegular() bool
 	IsSymlink() bool
+	Owner() int
+	Group() int
 }
 
 // FileMode is similar to os.FileMode

lib/model/folder_sendrecv.go

@@ -8,7 +8,6 @@ package model
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"math/rand"
 	"path/filepath"
@@ -17,6 +16,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/events"
@@ -587,6 +588,11 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 				return err
 			}
 
+			// Copy the parent owner and group, if we are supposed to do that.
+			if err := f.maybeCopyOwner(path); err != nil {
+				return err
+			}
+
 			// Stat the directory so we can check its permissions.
 			info, err := f.fs.Lstat(path)
 			if err != nil {
@@ -707,7 +713,10 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 	// We declare a function that acts on only the path name, so
 	// we can pass it to InWritableDir.
 	createLink := func(path string) error {
-		return f.fs.CreateSymlink(file.SymlinkTarget, path)
+		if err := f.fs.CreateSymlink(file.SymlinkTarget, path); err != nil {
+			return err
+		}
+		return f.maybeCopyOwner(path)
 	}
 
 	if err = osutil.InWritableDir(createLink, f.fs, file.Name); err == nil {
@@ -1433,6 +1442,11 @@ func (f *sendReceiveFolder) performFinish(ignores *ignore.Matcher, file, curFile
 		}
 	}
 
+	// Copy the parent owner and group, if we are supposed to do that.
+	if err := f.maybeCopyOwner(tempName); err != nil {
+		return err
+	}
+
 	if stat, err := f.fs.Lstat(file.Name); err == nil {
 		// There is an old file or directory already in place. We need to
 		// handle that.
@@ -1888,6 +1902,26 @@ func (f *sendReceiveFolder) checkToBeDeleted(cur protocol.FileInfo, scanChan cha
 	return nil
 }
 
+func (f *sendReceiveFolder) maybeCopyOwner(path string) error {
+	if !f.CopyOwnershipFromParent {
+		// Not supposed to do anything.
+		return nil
+	}
+	if runtime.GOOS == "windows" {
+		// Can't do anything.
+		return nil
+	}
+
+	info, err := f.fs.Lstat(filepath.Dir(path))
+	if err != nil {
+		return errors.Wrap(err, "copy owner from parent")
+	}
+	if err := f.fs.Lchown(path, info.Owner(), info.Group()); err != nil {
+		return errors.Wrap(err, "copy owner from parent")
+	}
+	return nil
+}
+
 // A []FileError is sent as part of an event and will be JSON serialized.
 type FileError struct {
 	Path string `json:"path"`

lib/model/folder_sendrecv_test.go

@@ -14,6 +14,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -706,7 +707,7 @@ func TestDiffEmpty(t *testing.T) {
 // option is true and the permissions do not match between the file on disk and
 // in the db.
 func TestDeleteIgnorePerms(t *testing.T) {
-	m := setUpModel(protocol.FileInfo{})
+	m := setUpModel()
 	f := setUpSendReceiveFolder(m)
 	f.IgnorePerms = true
 
@@ -743,3 +744,119 @@ func TestDeleteIgnorePerms(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestCopyOwner(t *testing.T) {
+	// Verifies that owner and group are copied from the parent, for both
+	// files and directories.
+
+	if runtime.GOOS == "windows" {
+		t.Skip("copying owner not supported on Windows")
+	}
+
+	const (
+		expOwner = 1234
+		expGroup = 5678
+	)
+
+	// Set up a folder with the CopyParentOwner bit and backed by a fake
+	// filesystem.
+
+	m := setUpModel()
+	f := &sendReceiveFolder{
+		folder: folder{
+			stateTracker:        newStateTracker("default"),
+			model:               m,
+			initialScanFinished: make(chan struct{}),
+			ctx:                 context.TODO(),
+			FolderConfiguration: config.FolderConfiguration{
+				FilesystemType:          fs.FilesystemTypeFake,
+				Path:                    "/TestCopyOwner",
+				CopyOwnershipFromParent: true,
+			},
+		},
+
+		queue:         newJobQueue(),
+		pullErrors:    make(map[string]string),
+		pullErrorsMut: sync.NewMutex(),
+	}
+
+	f.fs = f.Filesystem()
+
+	// Create a parent dir with a certain owner/group.
+
+	f.fs.Mkdir("foo", 0755)
+	f.fs.Lchown("foo", expOwner, expGroup)
+
+	dir := protocol.FileInfo{
+		Name:        "foo/bar",
+		Type:        protocol.FileInfoTypeDirectory,
+		Permissions: 0755,
+	}
+
+	// Have the folder create a subdirectory, verify that it's the correct
+	// owner/group.
+
+	dbUpdateChan := make(chan dbUpdateJob, 1)
+	defer close(dbUpdateChan)
+	f.handleDir(dir, dbUpdateChan)
+	<-dbUpdateChan // empty the channel for later
+
+	info, err := f.fs.Lstat("foo/bar")
+	if err != nil {
+		t.Fatal("Unexpected error (dir):", err)
+	}
+	if info.Owner() != expOwner || info.Group() != expGroup {
+		t.Fatalf("Expected dir owner/group to be %d/%d, not %d/%d", expOwner, expGroup, info.Owner(), info.Group())
+	}
+
+	// Have the folder create a file, verify it's the correct owner/group.
+	// File is zero sized to avoid having to handle copies/pulls.
+
+	file := protocol.FileInfo{
+		Name:        "foo/bar/baz",
+		Type:        protocol.FileInfoTypeFile,
+		Permissions: 0644,
+	}
+
+	// Wire some stuff. The flow here is handleFile() -[copierChan]->
+	// copierRoutine() -[finisherChan]-> finisherRoutine() -[dbUpdateChan]->
+	// back to us and we're done. The copier routine doesn't do anything,
+	// but it's the way data is passed around. When the database update
+	// comes the finisher is done.
+
+	finisherChan := make(chan *sharedPullerState)
+	defer close(finisherChan)
+	copierChan := make(chan copyBlocksState)
+	defer close(copierChan)
+	go f.copierRoutine(copierChan, nil, finisherChan)
+	go f.finisherRoutine(nil, finisherChan, dbUpdateChan, nil)
+	f.handleFile(file, copierChan, nil, nil)
+	<-dbUpdateChan
+
+	info, err = f.fs.Lstat("foo/bar/baz")
+	if err != nil {
+		t.Fatal("Unexpected error (file):", err)
+	}
+	if info.Owner() != expOwner || info.Group() != expGroup {
+		t.Fatalf("Expected file owner/group to be %d/%d, not %d/%d", expOwner, expGroup, info.Owner(), info.Group())
+	}
+
+	// Have the folder create a symlink. Verify it accordingly.
+	symlink := protocol.FileInfo{
+		Name:          "foo/bar/sym",
+		Type:          protocol.FileInfoTypeSymlink,
+		Permissions:   0644,
+		SymlinkTarget: "over the rainbow",
+	}
+
+	f.handleSymlink(symlink, dbUpdateChan)
+	<-dbUpdateChan
+
+	info, err = f.fs.Lstat("foo/bar/sym")
+	if err != nil {
+		t.Fatal("Unexpected error (file):", err)
+	}
+	if info.Owner() != expOwner || info.Group() != expGroup {
+		t.Fatalf("Expected symlink owner/group to be %d/%d, not %d/%d", expOwner, expGroup, info.Owner(), info.Group())
+	}
+}

lib/scanner/virtualfs_test.go

@@ -109,6 +109,8 @@ func (f fakeInfo) ModTime() time.Time { return time.Unix(1234567890, 0) }
 func (f fakeInfo) IsDir() bool        { return strings.Contains(filepath.Base(f.name), "dir") || f.name == "." }
 func (f fakeInfo) IsRegular() bool    { return !f.IsDir() }
 func (f fakeInfo) IsSymlink() bool    { return false }
+func (f fakeInfo) Owner() int         { return 0 }
+func (f fakeInfo) Group() int         { return 0 }
 
 type fakeFile struct {
 	name       string