perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Keep folder meta data persistently in db (fixes #4400)

Browse Source 
This keeps the data we need about sequence numbers and object counts
persistently in the database. The sizeTracker is expanded into a
metadataTracker than handled multiple folders, and the Counts struct is
made protobuf serializable. It gains a Sequence field to assist in
tracking that as well, and a collection of Counts become a CountsSet
(for serialization purposes).

The initial database scan is also a consistency check of the global
entries. This shouldn't strictly be necessary. Nonetheless I added a
created timestamp to the metadata and set a variable to compare against
that. When the time since the metadata creation is old enough, we drop
the metadata and rebuild from scratch like we used to, while also
consistency checking.

A new environment variable STCHECKDBEVERY can override this interval,
and for example be set to zero to force the check immediately.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4547
LGTM: imsodin
This commit is contained in:
Jakob Borg
2017-12-14 09:51:17 +00:00
committed by Audrius Butkevicius
parent 8c91ced784
commit d1d967f0cf
14 changed files with 908 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lib/db/leveldb_transactions.go
View File

@@ -85,7 +85,7 @@ func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.Fi
// updateGlobal adds this device+version to the version list for the given
// file. If the device is already present in the list, the version is updated.
// If the file does not have an entry in the global list, it is created.
func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.FileInfo, globalSize *sizeTracker) bool {
func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.FileInfo, meta *metadataTracker) bool {
l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.Invalid)
name := []byte(file.Name)
gk := t.db.globalKey(folder, name)
@@ -106,7 +106,7 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
if i == 0 {
// Keep the current newest file around so we can subtract it from
// the globalSize if we replace it.
// the metadata if we replace it.
oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
}
@@ -169,16 +169,16 @@ insert:
// We just inserted a new newest version. Fixup the global size
// calculation.
if !file.Version.Equal(oldFile.Version) {
globalSize.addFile(file)
meta.addFile(globalDeviceID, file)
if hasOldFile {
// We have the old file that was removed at the head of the list.
globalSize.removeFile(oldFile)
meta.removeFile(globalDeviceID, oldFile)
} else if len(fl.Versions) > 1 {
// The previous newest version is now at index 1, grab it from there.
if oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
globalSize.removeFile(oldFile)
meta.removeFile(globalDeviceID, oldFile)
}
}
}
@@ -193,7 +193,7 @@ insert:
// removeFromGlobal removes the device from the global version list for the
// given file. If the version list is empty after this, the file entry is
// removed entirely.
func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, globalSize *sizeTracker) {
func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, meta *metadataTracker) {
l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
gk := t.db.globalKey(folder, file)
@@ -214,13 +214,13 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
removed := false
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if i == 0 && globalSize != nil {
if i == 0 && meta != nil {
f, ok := t.getFile(folder, device, file)
if !ok {
// didn't exist anyway, apparently
continue
}
globalSize.removeFile(f)
meta.removeFile(globalDeviceID, f)
removed = true
}
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
@@ -238,7 +238,7 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
globalSize.addFile(f)
meta.addFile(globalDeviceID, f)
}
}
}