cmd/syncthing, lib/db, lib/model, lib/protocol: Implement delta indexes (fixes #438)
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3427
This commit is contained in:
Jakob Borg
committed by
Audrius Butkevicius
Audrius Butkevicius
parent
8ab6b60778
commit
47fa4b0a2c
102
lib/db/set.go
102
lib/db/set.go
@@ -14,6 +14,7 @@ package db
|
||||
|
||||
import (
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
@@ -21,13 +22,15 @@ import (
|
||||
)
|
||||
|
||||
type FileSet struct {
|
||||
localVersion map[protocol.DeviceID]int64
|
||||
mutex sync.Mutex
|
||||
localVersion int64 // Our local version counter
|
||||
folder string
|
||||
db *Instance
|
||||
blockmap *BlockMap
|
||||
localSize sizeTracker
|
||||
globalSize sizeTracker
|
||||
|
||||
remoteLocalVersion map[protocol.DeviceID]int64 // Highest seen local versions for other devices
|
||||
rlvMutex sync.Mutex // protects remoteLocalVersion
|
||||
}
|
||||
|
||||
// FileIntf is the set of methods implemented by both protocol.FileInfo and
|
||||
@@ -95,11 +98,11 @@ func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
|
||||
|
||||
func NewFileSet(folder string, db *Instance) *FileSet {
|
||||
var s = FileSet{
|
||||
localVersion: make(map[protocol.DeviceID]int64),
|
||||
folder: folder,
|
||||
db: db,
|
||||
blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
|
||||
mutex: sync.NewMutex(),
|
||||
remoteLocalVersion: make(map[protocol.DeviceID]int64),
|
||||
folder: folder,
|
||||
db: db,
|
||||
blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
|
||||
rlvMutex: sync.NewMutex(),
|
||||
}
|
||||
|
||||
s.db.checkGlobals([]byte(folder), &s.globalSize)
|
||||
@@ -107,16 +110,17 @@ func NewFileSet(folder string, db *Instance) *FileSet {
|
||||
var deviceID protocol.DeviceID
|
||||
s.db.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool {
|
||||
copy(deviceID[:], device)
|
||||
if f.LocalVersion > s.localVersion[deviceID] {
|
||||
s.localVersion[deviceID] = f.LocalVersion
|
||||
}
|
||||
if deviceID == protocol.LocalDeviceID {
|
||||
if f.LocalVersion > s.localVersion {
|
||||
s.localVersion = f.LocalVersion
|
||||
}
|
||||
s.localSize.addFile(f)
|
||||
} else if f.LocalVersion > s.remoteLocalVersion[deviceID] {
|
||||
s.remoteLocalVersion[deviceID] = f.LocalVersion
|
||||
}
|
||||
return true
|
||||
})
|
||||
l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
|
||||
clock(s.localVersion[protocol.LocalDeviceID])
|
||||
|
||||
return &s
|
||||
}
|
||||
@@ -124,13 +128,23 @@ func NewFileSet(folder string, db *Instance) *FileSet {
|
||||
func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
l.Debugf("%s Replace(%v, [%d])", s.folder, device, len(fs))
|
||||
normalizeFilenames(fs)
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
s.localVersion[device] = s.db.replace([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
|
||||
if len(fs) == 0 {
|
||||
// Reset the local version if all files were removed.
|
||||
s.localVersion[device] = 0
|
||||
if device == protocol.LocalDeviceID {
|
||||
if len(fs) == 0 {
|
||||
s.localVersion = 0
|
||||
} else {
|
||||
// Always overwrite LocalVersion on updated files to ensure
|
||||
// correct ordering. The caller is supposed to leave it set to
|
||||
// zero anyhow.
|
||||
for i := range fs {
|
||||
fs[i].LocalVersion = atomic.AddInt64(&s.localVersion, 1)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.rlvMutex.Lock()
|
||||
s.remoteLocalVersion[device] = maxLocalVersion(fs)
|
||||
s.rlvMutex.Unlock()
|
||||
}
|
||||
s.db.replace([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
|
||||
if device == protocol.LocalDeviceID {
|
||||
s.blockmap.Drop()
|
||||
s.blockmap.Add(fs)
|
||||
@@ -140,12 +154,11 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs))
|
||||
normalizeFilenames(fs)
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
if device == protocol.LocalDeviceID {
|
||||
discards := make([]protocol.FileInfo, 0, len(fs))
|
||||
updates := make([]protocol.FileInfo, 0, len(fs))
|
||||
for _, newFile := range fs {
|
||||
for i, newFile := range fs {
|
||||
fs[i].LocalVersion = atomic.AddInt64(&s.localVersion, 1)
|
||||
existingFile, ok := s.db.getFile([]byte(s.folder), device[:], []byte(newFile.Name))
|
||||
if !ok || !existingFile.Version.Equal(newFile.Version) {
|
||||
discards = append(discards, existingFile)
|
||||
@@ -154,10 +167,12 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
}
|
||||
s.blockmap.Discard(discards)
|
||||
s.blockmap.Update(updates)
|
||||
} else {
|
||||
s.rlvMutex.Lock()
|
||||
s.remoteLocalVersion[device] = maxLocalVersion(fs)
|
||||
s.rlvMutex.Unlock()
|
||||
}
|
||||
if lv := s.db.updateFiles([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize); lv > s.localVersion[device] {
|
||||
s.localVersion[device] = lv
|
||||
}
|
||||
s.db.updateFiles([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
|
||||
}
|
||||
|
||||
func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
|
||||
@@ -230,9 +245,13 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
|
||||
}
|
||||
|
||||
func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
return s.localVersion[device]
|
||||
if device == protocol.LocalDeviceID {
|
||||
return atomic.LoadInt64(&s.localVersion)
|
||||
}
|
||||
|
||||
s.rlvMutex.Lock()
|
||||
defer s.rlvMutex.Unlock()
|
||||
return s.remoteLocalVersion[device]
|
||||
}
|
||||
|
||||
func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
|
||||
@@ -243,6 +262,37 @@ func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
|
||||
return s.globalSize.Size()
|
||||
}
|
||||
|
||||
func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {
|
||||
id := s.db.getIndexID(device[:], []byte(s.folder))
|
||||
if id == 0 && device == protocol.LocalDeviceID {
|
||||
// No index ID set yet. We create one now.
|
||||
id = protocol.NewIndexID()
|
||||
s.db.setIndexID(device[:], []byte(s.folder), id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
|
||||
if device == protocol.LocalDeviceID {
|
||||
panic("do not explicitly set index ID for local device")
|
||||
}
|
||||
s.db.setIndexID(device[:], []byte(s.folder), id)
|
||||
}
|
||||
|
||||
// maxLocalVersion returns the highest of the LocalVersion numbers found in
|
||||
// the given slice of FileInfos. This should really be the LocalVersion of
|
||||
// the last item, but Syncthing v0.14.0 and other implementations may not
|
||||
// implement update sorting....
|
||||
func maxLocalVersion(fs []protocol.FileInfo) int64 {
|
||||
var max int64
|
||||
for _, f := range fs {
|
||||
if f.LocalVersion > max {
|
||||
max = f.LocalVersion
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// DropFolder clears out all information related to the given folder from the
|
||||
// database.
|
||||
func DropFolder(db *Instance, folder string) {
|
||||
|
||||
Reference in New Issue
Block a user