diff --git a/internal/files/blockmap.go b/internal/files/blockmap.go index d47fe9db..618f44f4 100644 --- a/internal/files/blockmap.go +++ b/internal/files/blockmap.go @@ -29,6 +29,7 @@ import ( "sync" "github.com/syncthing/syncthing/internal/config" + "github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/protocol" "github.com/syndtr/goleveldb/leveldb" @@ -171,7 +172,7 @@ func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) b for iter.Next() && iter.Error() == nil { folder, file := fromBlockKey(iter.Key()) index := binary.BigEndian.Uint32(iter.Value()) - if iterFn(folder, nativeFilename(file), index) { + if iterFn(folder, osutil.NativeFilename(file), index) { return true } } diff --git a/internal/files/set.go b/internal/files/set.go index 01cc517c..730a55d8 100644 --- a/internal/files/set.go +++ b/internal/files/set.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/syncthing/syncthing/internal/lamport" + "github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/protocol" "github.com/syndtr/goleveldb/leveldb" ) @@ -174,19 +175,19 @@ func (s *Set) WithGlobalTruncated(fn fileIterator) { } func (s *Set) Get(device protocol.DeviceID, file string) protocol.FileInfo { - f := ldbGet(s.db, []byte(s.folder), device[:], []byte(normalizedFilename(file))) - f.Name = nativeFilename(f.Name) + f := ldbGet(s.db, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))) + f.Name = osutil.NativeFilename(f.Name) return f } func (s *Set) GetGlobal(file string) protocol.FileInfo { - f := ldbGetGlobal(s.db, []byte(s.folder), []byte(normalizedFilename(file))) - f.Name = nativeFilename(f.Name) + f := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file))) + f.Name = osutil.NativeFilename(f.Name) return f } func (s *Set) Availability(file string) []protocol.DeviceID { - return ldbAvailability(s.db, []byte(s.folder), []byte(normalizedFilename(file))) + return ldbAvailability(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file))) } func (s *Set) LocalVersion(device protocol.DeviceID) uint64 { @@ -213,7 +214,7 @@ func DropFolder(db *leveldb.DB, folder string) { func normalizeFilenames(fs []protocol.FileInfo) { for i := range fs { - fs[i].Name = normalizedFilename(fs[i].Name) + fs[i].Name = osutil.NormalizedFilename(fs[i].Name) } } @@ -221,10 +222,10 @@ func nativeFileIterator(fn fileIterator) fileIterator { return func(fi protocol.FileIntf) bool { switch f := fi.(type) { case protocol.FileInfo: - f.Name = nativeFilename(f.Name) + f.Name = osutil.NativeFilename(f.Name) return fn(f) case protocol.FileInfoTruncated: - f.Name = nativeFilename(f.Name) + f.Name = osutil.NativeFilename(f.Name) return fn(f) default: panic("unknown interface type") diff --git a/internal/model/model.go b/internal/model/model.go index 2927490e..73d10b60 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -39,6 +39,7 @@ import ( "github.com/syncthing/syncthing/internal/protocol" "github.com/syncthing/syncthing/internal/scanner" "github.com/syncthing/syncthing/internal/stats" + "github.com/syncthing/syncthing/internal/symlinks" "github.com/syncthing/syncthing/internal/versioner" "github.com/syndtr/goleveldb/leveldb" ) @@ -114,6 +115,8 @@ type Model struct { var ( ErrNoSuchFile = errors.New("no such file") ErrInvalid = errors.New("file is invalid") + + SymlinkWarning = sync.Once{} ) // NewModel creates and starts a new model. The model starts in read-only mode, @@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F for i := 0; i < len(fs); { lamport.Default.Tick(fs[i].Version) - if ignores != nil && ignores.Match(fs[i].Name) { + if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) { if debug { - l.Debugln("dropping update for ignored", fs[i]) + l.Debugln("dropping update for ignored/unsupported symlink", fs[i]) } fs[i] = fs[len(fs)-1] fs = fs[:len(fs)-1] @@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot for i := 0; i < len(fs); { lamport.Default.Tick(fs[i].Version) - if ignores != nil && ignores.Match(fs[i].Name) { + if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) { if debug { - l.Debugln("dropping update for ignored", fs[i]) + l.Debugln("dropping update for ignored/unsupported symlink", fs[i]) } fs[i] = fs[len(fs)-1] fs = fs[:len(fs)-1] @@ -655,7 +658,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset } lf := r.Get(protocol.LocalDeviceID, name) - if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) { + if lf.IsInvalid() || lf.IsDeleted() { if debug { l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, size, lf) } @@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset m.fmut.RLock() fn := filepath.Join(m.folderCfgs[folder].Path, name) m.fmut.RUnlock() - fd, err := os.Open(fn) // XXX: Inefficient, should cache fd? - if err != nil { - return nil, err + + var reader io.ReaderAt + var err error + if lf.IsSymlink() { + target, _, err := symlinks.Read(fn) + if err != nil { + return nil, err + } + reader = strings.NewReader(target) + } else { + reader, err = os.Open(fn) // XXX: Inefficient, should cache fd? + if err != nil { + return nil, err + } + + defer reader.(*os.File).Close() } - defer fd.Close() buf := make([]byte, size) - _, err = fd.ReadAt(buf, offset) + _, err = reader.ReadAt(buf, offset) if err != nil { return nil, err } @@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol maxLocalVer = f.LocalVersion } - if ignores != nil && ignores.Match(f.Name) { + if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) { if debug { - l.Debugln("not sending update for ignored", f) + l.Debugln("not sending update for ignored/unsupported symlink", f) } return true } @@ -1085,7 +1100,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error { } seenPrefix = true - if !protocol.IsDeleted(f.Flags) { + if !f.IsDeleted() { if f.IsInvalid() { return true } @@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error { batch = batch[:0] } - if ignores != nil && ignores.Match(f.Name) { - // File has been ignored. Set invalid bit. + if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) { + // File has been ignored or an unsupported symlink. Set invalid bit. l.Debugln("setting invalid bit on ignored", f) nf := protocol.FileInfo{ Name: f.Name, @@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error { "size": f.Size(), }) batch = append(batch, nf) - } else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) { + } else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) { // File has been deleted nf := protocol.FileInfo{ Name: f.Name, @@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() { } } } + +func symlinkInvalid(isLink bool) bool { + if !symlinks.Supported && isLink { + SymlinkWarning.Do(func() { + l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.") + }) + return true + } + return false +} diff --git a/internal/model/puller.go b/internal/model/puller.go index 976995b5..d4fbdc8c 100644 --- a/internal/model/puller.go +++ b/internal/model/puller.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "sync" @@ -32,6 +33,7 @@ import ( "github.com/syncthing/syncthing/internal/osutil" "github.com/syncthing/syncthing/internal/protocol" "github.com/syncthing/syncthing/internal/scanner" + "github.com/syncthing/syncthing/internal/symlinks" "github.com/syncthing/syncthing/internal/versioner" ) @@ -313,15 +315,16 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo } switch { - case protocol.IsDeleted(file.Flags): - // A deleted file or directory + case file.IsDeleted(): + // A deleted file, directory or symlink deletions = append(deletions, file) - case protocol.IsDirectory(file.Flags): + case file.IsDirectory() && !file.IsSymlink(): // A new or changed directory p.handleDir(file) default: - // A new or changed file. This is the only case where we do stuff - // in the background; the other three are done synchronously. + // A new or changed file or symlink. This is the only case where we + // do stuff in the background; the other three are done + // synchronously. p.handleFile(file, copyChan, finisherChan) } @@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) { func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) { curFile := p.model.CurrentFolderFile(p.folder, file.Name) - if len(curFile.Blocks) == len(file.Blocks) { - for i := range file.Blocks { - if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) { - goto FilesAreDifferent - } - } + if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) { // We are supposed to copy the entire file, and then fetch nothing. We // are only updating metadata, so we don't actually *need* to make the // copy. if debug { l.Debugln(p, "taking shortcut on", file.Name) } - p.shortcutFile(file) + if file.IsSymlink() { + p.shortcutSymlink(curFile, file) + } else { + p.shortcutFile(file) + } return } -FilesAreDifferent: - scanner.PopulateOffsets(file.Blocks) // Figure out the absolute filenames we need once and for all @@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) { p.model.updateLocal(p.folder, file) } +// shortcutSymlink changes the symlinks type if necessery. +func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) { + err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags) + if err != nil { + l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err) + return + } + + p.model.updateLocal(p.folder, file) +} + // copierRoutine reads copierStates until the in channel closes and performs // the relevant copies when possible, or passes it to the puller routine. func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) { @@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) { continue } + // If it's a symlink, the target of the symlink is inside the file. + if state.file.IsSymlink() { + content, err := ioutil.ReadFile(state.realName) + if err != nil { + l.Warnln("puller: final: reading symlink:", err) + continue + } + + // Remove the file, and replace it with a symlink. + err = osutil.InWritableDir(func(path string) error { + os.Remove(path) + return symlinks.Create(path, string(content), state.file.Flags) + }, state.realName) + if err != nil { + l.Warnln("puller: final: creating symlink:", err) + continue + } + } + // Record the updated file in the index p.model.updateLocal(p.folder, state.file) } diff --git a/internal/files/filenames_darwin.go b/internal/osutil/filenames_darwin.go similarity index 89% rename from internal/files/filenames_darwin.go rename to internal/osutil/filenames_darwin.go index 4d2559cc..ce8c4085 100644 --- a/internal/files/filenames_darwin.go +++ b/internal/osutil/filenames_darwin.go @@ -13,14 +13,14 @@ // You should have received a copy of the GNU General Public License along // with this program. If not, see . -package files +package osutil import "code.google.com/p/go.text/unicode/norm" -func normalizedFilename(s string) string { +func NormalizedFilename(s string) string { return norm.NFC.String(s) } -func nativeFilename(s string) string { +func NativeFilename(s string) string { return norm.NFD.String(s) } diff --git a/internal/files/filenames_unix.go b/internal/osutil/filenames_unix.go similarity index 89% rename from internal/files/filenames_unix.go rename to internal/osutil/filenames_unix.go index 573069c1..7741f617 100644 --- a/internal/files/filenames_unix.go +++ b/internal/osutil/filenames_unix.go @@ -15,14 +15,14 @@ // +build !windows,!darwin -package files +package osutil import "code.google.com/p/go.text/unicode/norm" -func normalizedFilename(s string) string { +func NormalizedFilename(s string) string { return norm.NFC.String(s) } -func nativeFilename(s string) string { +func NativeFilename(s string) string { return s } diff --git a/internal/files/filenames_windows.go b/internal/osutil/filenames_windows.go similarity index 89% rename from internal/files/filenames_windows.go rename to internal/osutil/filenames_windows.go index 34a37a9e..88ba71ba 100644 --- a/internal/files/filenames_windows.go +++ b/internal/osutil/filenames_windows.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License along // with this program. If not, see . -package files +package osutil import ( "path/filepath" @@ -21,10 +21,10 @@ import ( "code.google.com/p/go.text/unicode/norm" ) -func normalizedFilename(s string) string { +func NormalizedFilename(s string) string { return norm.NFC.String(filepath.ToSlash(s)) } -func nativeFilename(s string) string { +func NativeFilename(s string) string { return filepath.FromSlash(s) } diff --git a/internal/protocol/message.go b/internal/protocol/message.go index eff837fe..355050ca 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -37,7 +37,7 @@ func (f FileInfo) String() string { } func (f FileInfo) Size() (bytes int64) { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } for _, b := range f.Blocks { @@ -47,15 +47,23 @@ func (f FileInfo) Size() (bytes int64) { } func (f FileInfo) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfo) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 } func (f FileInfo) IsDirectory() bool { - return IsDirectory(f.Flags) + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfo) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + +func (f FileInfo) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } // Used for unmarshalling a FileInfo structure but skipping the actual block list @@ -75,7 +83,7 @@ func (f FileInfoTruncated) String() string { // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } if f.NumBlocks < 2 { @@ -86,17 +94,32 @@ func (f FileInfoTruncated) Size() int64 { } func (f FileInfoTruncated) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfoTruncated) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 +} + +func (f FileInfoTruncated) IsDirectory() bool { + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfoTruncated) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + +func (f FileInfoTruncated) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } type FileIntf interface { Size() int64 IsDeleted() bool IsInvalid() bool + IsDirectory() bool + IsSymlink() bool + HasPermissionBits() bool } type BlockInfo struct { diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go index ae7f480b..fb51a1c0 100644 --- a/internal/protocol/protocol.go +++ b/internal/protocol/protocol.go @@ -49,10 +49,14 @@ const ( ) const ( - FlagDeleted uint32 = 1 << 12 - FlagInvalid = 1 << 13 - FlagDirectory = 1 << 14 - FlagNoPermBits = 1 << 15 + FlagDeleted uint32 = 1 << 12 + FlagInvalid = 1 << 13 + FlagDirectory = 1 << 14 + FlagNoPermBits = 1 << 15 + FlagSymlink = 1 << 16 + FlagSymlinkMissingTarget = 1 << 17 + + SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) const ( @@ -637,19 +641,3 @@ func (c *rawConnection) Statistics() Statistics { OutBytesTotal: c.cw.Tot(), } } - -func IsDeleted(bits uint32) bool { - return bits&FlagDeleted != 0 -} - -func IsInvalid(bits uint32) bool { - return bits&FlagInvalid != 0 -} - -func IsDirectory(bits uint32) bool { - return bits&FlagDirectory != 0 -} - -func HasPermissionBits(bits uint32) bool { - return bits&FlagNoPermBits == 0 -} diff --git a/internal/scanner/blockqueue.go b/internal/scanner/blockqueue.go index 9d75dac7..2d1a8656 100644 --- a/internal/scanner/blockqueue.go +++ b/internal/scanner/blockqueue.go @@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) { func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) { for f := range inbox { - if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) { + if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() { outbox <- f continue } diff --git a/internal/scanner/blocks.go b/internal/scanner/blocks.go index 825c1250..2f28b1fe 100644 --- a/internal/scanner/blocks.go +++ b/internal/scanner/blocks.go @@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error { return nil } + +// BlockEqual returns whether two slices of blocks are exactly the same hash +// and index pair wise. +func BlocksEqual(src, tgt []protocol.BlockInfo) bool { + if len(tgt) != len(src) { + return false + } + + for i, sblk := range src { + if !bytes.Equal(sblk.Hash, tgt[i].Hash) { + return false + } + } + return true +} diff --git a/internal/scanner/walk.go b/internal/scanner/walk.go index 927a41e8..f2efda59 100644 --- a/internal/scanner/walk.go +++ b/internal/scanner/walk.go @@ -27,6 +27,7 @@ import ( "github.com/syncthing/syncthing/internal/ignore" "github.com/syncthing/syncthing/internal/lamport" "github.com/syncthing/syncthing/internal/protocol" + "github.com/syncthing/syncthing/internal/symlinks" ) type Walker struct { @@ -131,11 +132,75 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun return nil } + // We must perform this check, as symlinks on Windows are always + // .IsRegular or .IsDir unlike on Unix. + // Index wise symlinks are always files, regardless of what the target + // is, because symlinks carry their target path as their content. + isSymlink, _ := symlinks.IsSymlink(p) + if isSymlink { + var rval error + // If the target is a directory, do NOT descend down there. + // This will cause files to get tracked, and removing the symlink + // will as a result remove files in their real location. + // But do not SkipDir if the target is not a directory, as it will + // stop scanning the current directory. + if info.IsDir() { + rval = filepath.SkipDir + } + + // We always rehash symlinks as they have no modtime or + // permissions. + // We check if they point to the old target by checking that + // their existing blocks match with the blocks in the index. + // If we don't have a filer or don't support symlinks, skip. + if w.CurrentFiler == nil || !symlinks.Supported { + return rval + } + + target, flags, err := symlinks.Read(p) + flags = flags & protocol.SymlinkTypeMask + if err != nil { + if debug { + l.Debugln("readlink error:", p, err) + } + return rval + } + + blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0) + if err != nil { + if debug { + l.Debugln("hash link error:", p, err) + } + return rval + } + + cf := w.CurrentFiler.CurrentFile(rn) + if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) { + return rval + } + + f := protocol.FileInfo{ + Name: rn, + Version: lamport.Default.Tick(0), + Flags: protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666, + Modified: 0, + Blocks: blocks, + } + + if debug { + l.Debugln("symlink to hash:", p, f) + } + + fchan <- f + + return rval + } + if info.Mode().IsDir() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) - permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) - if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged { + permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode())) + if !cf.IsDeleted() && cf.IsDirectory() && permUnchanged { return nil } } @@ -162,8 +227,8 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun if info.Mode().IsRegular() { if w.CurrentFiler != nil { cf := w.CurrentFiler.CurrentFile(rn) - permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode())) - if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged { + permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode())) + if !cf.IsDeleted() && cf.Modified == info.ModTime().Unix() && permUnchanged { return nil } @@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool { return a&0777 == b&0777 } } + +// If the target is missing, Unix never knows what type of symlink it is +// and Windows always knows even if there is no target. +// Which means that without this special check a Unix node would be fighting +// with a Windows node about whether or not the target is known. +// Basically, if you don't know and someone else knows, just accept it. +// The fact that you don't know means you are on Unix, and on Unix you don't +// really care what the target type is. The moment you do know, and if something +// doesn't match, that will propogate throught the cluster. +func SymlinkTypeEqual(disk, index uint32) bool { + if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 { + return true + } + return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask + +} diff --git a/internal/symlinks/symlink_unix.go b/internal/symlinks/symlink_unix.go new file mode 100644 index 00000000..814c7411 --- /dev/null +++ b/internal/symlinks/symlink_unix.go @@ -0,0 +1,58 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +// +build !windows + +package symlinks + +import ( + "os" + + "github.com/syncthing/syncthing/internal/osutil" + "github.com/syncthing/syncthing/internal/protocol" +) + +var ( + Supported = true +) + +func Read(path string) (string, uint32, error) { + var mode uint32 + stat, err := os.Stat(path) + if err != nil { + mode = protocol.FlagSymlinkMissingTarget + } else if stat.IsDir() { + mode = protocol.FlagDirectory + } + path, err = os.Readlink(path) + + return osutil.NormalizedFilename(path), mode, err +} + +func IsSymlink(path string) (bool, error) { + lstat, err := os.Lstat(path) + if err != nil { + return false, err + } + return lstat.Mode()&os.ModeSymlink != 0, nil +} + +func Create(source, target string, flags uint32) error { + return os.Symlink(osutil.NativeFilename(target), source) +} + +func ChangeType(path string, flags uint32) error { + return nil +} diff --git a/internal/symlinks/symlink_windows.go b/internal/symlinks/symlink_windows.go new file mode 100644 index 00000000..1a99e3db --- /dev/null +++ b/internal/symlinks/symlink_windows.go @@ -0,0 +1,203 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +// +build windows + +package symlinks + +import ( + "os" + "path/filepath" + + "github.com/syncthing/syncthing/internal/osutil" + "github.com/syncthing/syncthing/internal/protocol" + + "syscall" + "unicode/utf16" + "unsafe" +) + +const ( + FSCTL_GET_REPARSE_POINT = 0x900a8 + FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 + FILE_ATTRIBUTE_REPARSE_POINT = 0x400 + IO_REPARSE_TAG_SYMLINK = 0xA000000C + SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1 +) + +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procDeviceIoControl = modkernel32.NewProc("DeviceIoControl") + procCreateSymbolicLink = modkernel32.NewProc("CreateSymbolicLinkW") + + Supported = false +) + +func init() { + // Needs administrator priviledges. + // Let's check that everything works. + // This could be done more officially: + // http://stackoverflow.com/questions/2094663/determine-if-windows-process-has-privilege-to-create-symbolic-link + // But I don't want to define 10 more structs just to look this up. + base := os.TempDir() + path := filepath.Join(base, "symlinktest") + defer os.Remove(path) + + err := Create(path, base, protocol.FlagDirectory) + if err != nil { + return + } + + isLink, err := IsSymlink(path) + if err != nil || !isLink { + return + } + + target, flags, err := Read(path) + if err != nil || osutil.NativeFilename(target) != base || flags&protocol.FlagDirectory == 0 { + return + } + Supported = true +} + +type reparseData struct { + reparseTag uint32 + reparseDataLength uint16 + reserved uint16 + substitueNameOffset uint16 + substitueNameLength uint16 + printNameOffset uint16 + printNameLength uint16 + flags uint32 + // substituteName - 264 widechars max = 528 bytes + // printName - 260 widechars max = 520 bytes + // = 1048 bytes total + buffer [1048]uint16 +} + +func (r *reparseData) PrintName() string { + // No clue why the offset and length is doubled... + offset := r.printNameOffset / 2 + length := r.printNameLength / 2 + return string(utf16.Decode(r.buffer[offset : offset+length])) +} + +func (r *reparseData) SubstituteName() string { + // No clue why the offset and length is doubled... + offset := r.substitueNameOffset / 2 + length := r.substitueNameLength / 2 + return string(utf16.Decode(r.buffer[offset : offset+length])) +} + +func Read(path string) (string, uint32, error) { + ptr, err := syscall.UTF16PtrFromString(path) + if err != nil { + return "", protocol.FlagSymlinkMissingTarget, err + } + handle, err := syscall.CreateFile(ptr, 0, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OPEN_REPARSE_POINT, 0) + if err != nil || handle == syscall.InvalidHandle { + return "", protocol.FlagSymlinkMissingTarget, err + } + defer syscall.Close(handle) + var ret uint16 + var data reparseData + + r1, _, err := syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), FSCTL_GET_REPARSE_POINT, 0, 0, uintptr(unsafe.Pointer(&data)), unsafe.Sizeof(data), uintptr(unsafe.Pointer(&ret)), 0, 0) + if r1 == 0 { + return "", protocol.FlagSymlinkMissingTarget, err + } + + var flags uint32 = 0 + attr, err := syscall.GetFileAttributes(ptr) + if err != nil { + flags = protocol.FlagSymlinkMissingTarget + } else if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + flags = protocol.FlagDirectory + } + + return osutil.NormalizedFilename(data.PrintName()), flags, nil +} + +func IsSymlink(path string) (bool, error) { + ptr, err := syscall.UTF16PtrFromString(path) + if err != nil { + return false, err + } + + attr, err := syscall.GetFileAttributes(ptr) + if err != nil { + return false, err + } + return attr&FILE_ATTRIBUTE_REPARSE_POINT != 0, nil +} + +func Create(source, target string, flags uint32) error { + srcp, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err + } + + trgp, err := syscall.UTF16PtrFromString(osutil.NativeFilename(target)) + if err != nil { + return err + } + + // Sadly for Windows we need to specify the type of the symlink, + // whether it's a directory symlink or a file symlink. + // If the flags doesn't reveal the target type, try to evaluate it + // ourselves, and worst case default to the symlink pointing to a file. + mode := 0 + if flags&protocol.FlagSymlinkMissingTarget != 0 { + path := target + if !filepath.IsAbs(target) { + path = filepath.Join(filepath.Dir(source), target) + } + + stat, err := os.Stat(path) + if err == nil && stat.IsDir() { + mode = SYMBOLIC_LINK_FLAG_DIRECTORY + } + } else if flags&protocol.FlagDirectory != 0 { + mode = SYMBOLIC_LINK_FLAG_DIRECTORY + } + + r0, _, err := syscall.Syscall(procCreateSymbolicLink.Addr(), 3, uintptr(unsafe.Pointer(srcp)), uintptr(unsafe.Pointer(trgp)), uintptr(mode)) + if r0 == 1 { + return nil + } + return err +} + +func ChangeType(path string, flags uint32) error { + target, cflags, err := Read(path) + if err != nil { + return err + } + // If it's the same type, nothing to do. + if cflags&protocol.SymlinkTypeMask == flags&protocol.SymlinkTypeMask { + return nil + } + + // If the actual type is unknown, but the new type is file, nothing to do + if cflags&protocol.FlagSymlinkMissingTarget != 0 && flags&protocol.FlagDirectory == 0 { + return nil + } + return osutil.InWritableDir(func(path string) error { + // It should be a symlink as well hence no need to change permissions on + // the file. + os.Remove(path) + return Create(path, target, flags) + }, path) +} diff --git a/protocol/PROTOCOL.md b/protocol/PROTOCOL.md index faf2cb58..7d31075f 100644 --- a/protocol/PROTOCOL.md +++ b/protocol/PROTOCOL.md @@ -439,7 +439,7 @@ The Flags field is made up of the following single bit flags: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Reserved |P|I|D| Unix Perm. & Mode | + | Reserved |U|S|P|I|D| Unix Perm. & Mode | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - The lower 12 bits hold the common Unix permission and mode bits. An @@ -461,7 +461,16 @@ The Flags field is made up of the following single bit flags: disregarded on files with this bit set. The permissions bits MUST be set to the octal value 0666. - - Bit 0 through 16 are reserved for future use and SHALL be set to + - Bit 16 ("S") is set when the file is a symbolic link. The block list + SHALL be of one or more blocks since the target of the symlink is + stored within the blocks of the file. + + - Bit 15 ("U") is set when the symbolic links target does not exist. + On systems where symbolic links have types, this bit being means + that the default file symlink SHALL be used. If this bit is unset + bit 19 will decide the type of symlink to be created. + + - Bit 0 through 14 are reserved for future use and SHALL be set to zero. The hash algorithm is implied by the Hash length. Currently, the hash diff --git a/test/common_test.go b/test/common_test.go index b5fc6a16..6a3275f6 100644 --- a/test/common_test.go +++ b/test/common_test.go @@ -30,6 +30,8 @@ import ( "os/exec" "path/filepath" "time" + + "github.com/syncthing/syncthing/internal/symlinks" ) func init() { @@ -355,7 +357,22 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) { } var f fileInfo - if info.IsDir() { + if ok, err := symlinks.IsSymlink(path); err == nil && ok { + f = fileInfo{ + name: rn, + mode: os.ModeSymlink, + } + + tgt, _, err := symlinks.Read(path) + if err != nil { + return err + } + h := md5.New() + h.Write([]byte(tgt)) + hash := h.Sum(nil) + + copy(f.hash[:], hash) + } else if info.IsDir() { f = fileInfo{ name: rn, mode: info.Mode(), diff --git a/test/symlink_test.go b/test/symlink_test.go new file mode 100644 index 00000000..1d100f04 --- /dev/null +++ b/test/symlink_test.go @@ -0,0 +1,280 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +// +build integration + +package integration_test + +import ( + "log" + "os" + "strings" + "testing" + "time" + + "github.com/syncthing/syncthing/internal/symlinks" +) + +func TestSymlinks(t *testing.T) { + log.Println("Cleaning...") + err := removeAll("s1", "s2", "h1/index", "h2/index") + if err != nil { + t.Fatal(err) + } + + log.Println("Generating files...") + err = generateFiles("s1", 100, 20, "../bin/syncthing") + if err != nil { + t.Fatal(err) + } + + // A file that we will replace with a symlink later + + fd, err := os.Create("s1/fileToReplace") + if err != nil { + t.Fatal(err) + } + fd.Close() + + // A directory that we will replace with a symlink later + + err = os.Mkdir("s1/dirToReplace", 0755) + if err != nil { + t.Fatal(err) + } + + // A file and a symlink to that file + + fd, err = os.Create("s1/file") + if err != nil { + t.Fatal(err) + } + fd.Close() + err = symlinks.Create("s1/fileLink", "file", 0) + if err != nil { + log.Fatal(err) + } + + // A directory and a symlink to that directory + + err = os.Mkdir("s1/dir", 0755) + if err != nil { + t.Fatal(err) + } + err = symlinks.Create("s1/dirLink", "dir", 0) + if err != nil { + log.Fatal(err) + } + + // A link to something in the repo that does not exist + + err = symlinks.Create("s1/noneLink", "does/not/exist", 0) + if err != nil { + log.Fatal(err) + } + + // A link we will replace with a file later + + err = symlinks.Create("s1/repFileLink", "does/not/exist", 0) + if err != nil { + log.Fatal(err) + } + + // A link we will replace with a directory later + + err = symlinks.Create("s1/repDirLink", "does/not/exist", 0) + if err != nil { + log.Fatal(err) + } + + // Verify that the files and symlinks sync to the other side + + log.Println("Syncing...") + + sender := syncthingProcess{ // id1 + log: "1.out", + argv: []string{"-home", "h1"}, + port: 8081, + apiKey: apiKey, + } + err = sender.start() + if err != nil { + t.Fatal(err) + } + + receiver := syncthingProcess{ // id2 + log: "2.out", + argv: []string{"-home", "h2"}, + port: 8082, + apiKey: apiKey, + } + err = receiver.start() + if err != nil { + sender.stop() + t.Fatal(err) + } + + for { + comp, err := sender.peerCompletion() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + time.Sleep(time.Second) + continue + } + sender.stop() + receiver.stop() + t.Fatal(err) + } + + curComp := comp[id2] + + if curComp == 100 { + sender.stop() + receiver.stop() + break + } + + time.Sleep(time.Second) + } + + sender.stop() + receiver.stop() + + log.Println("Comparing directories...") + err = compareDirectories("s1", "s2") + if err != nil { + t.Fatal(err) + } + + log.Println("Making some changes...") + + // Remove one symlink + + err = os.Remove("s1/fileLink") + if err != nil { + log.Fatal(err) + } + + // Change the target of another + + err = os.Remove("s1/dirLink") + if err != nil { + log.Fatal(err) + } + err = symlinks.Create("s1/dirLink", "file", 0) + if err != nil { + log.Fatal(err) + } + + // Replace one with a file + + err = os.Remove("s1/repFileLink") + if err != nil { + log.Fatal(err) + } + + fd, err = os.Create("s1/repFileLink") + if err != nil { + log.Fatal(err) + } + fd.Close() + + /* Currently fails, to be fixed with #80 + + // Replace one with a directory + + err = os.Remove("s1/repDirLink") + if err != nil { + log.Fatal(err) + } + + err = os.Mkdir("s1/repDirLink", 0755) + if err != nil { + log.Fatal(err) + } + */ + + // Replace a file with a symlink + + err = os.Remove("s1/fileToReplace") + if err != nil { + log.Fatal(err) + } + err = symlinks.Create("s1/fileToReplace", "somewhere/non/existent", 0) + if err != nil { + log.Fatal(err) + } + + /* Currently fails, to be fixed with #80 + + // Replace a directory with a symlink + + err = os.RemoveAll("s1/dirToReplace") + if err != nil { + log.Fatal(err) + } + err = symlinks.Create("s1/dirToReplace", "somewhere/non/existent", 0) + if err != nil { + log.Fatal(err) + } + */ + + // Sync these changes and recheck + + log.Println("Syncing...") + + err = sender.start() + if err != nil { + t.Fatal(err) + } + + err = receiver.start() + if err != nil { + sender.stop() + t.Fatal(err) + } + + for { + comp, err := sender.peerCompletion() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + time.Sleep(time.Second) + continue + } + sender.stop() + receiver.stop() + t.Fatal(err) + } + + curComp := comp[id2] + + if curComp == 100 { + sender.stop() + receiver.stop() + break + } + + time.Sleep(time.Second) + } + + sender.stop() + receiver.stop() + + log.Println("Comparing directories...") + err = compareDirectories("s1", "s2") + if err != nil { + t.Fatal(err) + } +}