diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index 732d6146..2b82b23d 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -13,7 +13,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -const dbVersion = 2 +const dbVersion = 3 const ( KeyTypeDevice = iota @@ -28,13 +28,14 @@ const ( KeyTypeFolderMeta KeyTypeMiscData KeyTypeSequence + KeyTypeNeed ) -func (l VersionList) String() string { +func (vl VersionList) String() string { var b bytes.Buffer var id protocol.DeviceID b.WriteString("{") - for i, v := range l.Versions { + for i, v := range vl.Versions { if i > 0 { b.WriteString(", ") } @@ -45,18 +46,90 @@ func (l VersionList) String() string { return b.String() } +// update brings the VersionList up to date with file. It returns the updated +// VersionList, a potentially removed old FileVersion and its index, as well as +// the index where the new FileVersion was inserted. +func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) { + removedAt, insertedAt = -1, -1 + for i, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + removedAt = i + removedFV = v + vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...) + break + } + } + + nv := FileVersion{ + Device: device, + Version: file.Version, + Invalid: file.Invalid, + } + for i, v := range vl.Versions { + switch v.Version.Compare(file.Version) { + case protocol.Equal: + if nv.Invalid { + continue + } + fallthrough + + case protocol.Lesser: + // The version at this point in the list is equal to or lesser + // ("older") than us. We insert ourselves in front of it. + vl = vl.insertAt(i, nv) + return vl, removedFV, removedAt, i + + case protocol.ConcurrentLesser, protocol.ConcurrentGreater: + // The version at this point is in conflict with us. We must pull + // the actual file metadata to determine who wins. If we win, we + // insert ourselves in front of the loser here. (The "Lesser" and + // "Greater" in the condition above is just based on the device + // IDs in the version vector, which is not the only thing we use + // to determine the winner.) + // + // A surprise missing file entry here is counted as a win for us. + if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) { + vl = vl.insertAt(i, nv) + return vl, removedFV, removedAt, i + } + } + } + + // We didn't find a position for an insert above, so append to the end. + vl.Versions = append(vl.Versions, nv) + + return vl, removedFV, removedAt, len(vl.Versions) - 1 +} + +func (vl VersionList) insertAt(i int, v FileVersion) VersionList { + vl.Versions = append(vl.Versions, FileVersion{}) + copy(vl.Versions[i+1:], vl.Versions[i:]) + vl.Versions[i] = v + return vl +} + +func (vl VersionList) Get(device []byte) (FileVersion, bool) { + for _, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + return v, true + } + } + + return FileVersion{}, false +} + type fileList []protocol.FileInfo -func (l fileList) Len() int { - return len(l) +func (fl fileList) Len() int { + return len(fl) } -func (l fileList) Swap(a, b int) { - l[a], l[b] = l[b], l[a] +func (fl fileList) Swap(a, b int) { + fl[a], fl[b] = fl[b], fl[a] } -func (l fileList) Less(a, b int) bool { - return l[a].Name < l[b].Name +func (fl fileList) Less(a, b int) bool { + return fl[a].Name < fl[b].Name } // Flush batches to disk when they contain this many records. diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index cd8f02cf..19dadbcf 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -103,6 +103,9 @@ func (db *Instance) UpdateSchema() { if prevVersion <= 1 { db.updateSchema1to2() } + if prevVersion <= 2 { + db.updateSchema2to3() + } l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion) miscDB.PutInt64("dbVersion", dbVersion) @@ -310,26 +313,32 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) { } func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) { - k := db.globalKey(folder, file) - t := db.newReadOnlyTransaction() defer t.close() - bs, err := t.Get(k, nil) + _, _, f, ok := db.getGlobalInto(t, nil, nil, folder, file, truncate) + return f, ok +} + +func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) { + gk = db.globalKeyInto(gk, folder, file) + + bs, err := t.Get(gk, nil) if err != nil { - return nil, false + return gk, dk, nil, false } vl, ok := unmarshalVersionList(bs) if !ok { - return nil, false + return gk, dk, nil, false } - if fi, ok := db.getFileTrunc(db.deviceKey(folder, vl.Versions[0].Device, file), truncate); ok { - return fi, true + dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file) + if fi, ok := db.getFileTrunc(dk, truncate); ok { + return gk, dk, fi, true } - return nil, false + return gk, dk, nil, false } func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) { @@ -409,6 +418,11 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID { } func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) { + if bytes.Equal(device, protocol.LocalDeviceID[:]) { + db.withNeedLocal(folder, truncate, fn) + return + } + t := db.newReadOnlyTransaction() defer t.close() @@ -422,22 +436,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) continue } - have := false // If we have the file, any version - need := false // If we have a lower version of the file - var haveFileVersion FileVersion - for _, v := range vl.Versions { - if bytes.Equal(v.Device, device) { - have = true - haveFileVersion = v - // XXX: This marks Concurrent (i.e. conflicting) changes as - // needs. Maybe we should do that, but it needs special - // handling in the puller. - need = !v.Version.GreaterEqual(vl.Versions[0].Version) - break - } - } - - if have && !need { + haveFV, have := vl.Get(device) + // XXX: This marks Concurrent (i.e. conflicting) changes as + // needs. Maybe we should do that, but it needs special + // handling in the puller. + if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) { continue } @@ -474,7 +477,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) break } - l.Debugf("need folder=%q device=%v name=%q need=%v have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveFileVersion.Invalid, haveFileVersion.Version, needVersion, needDevice) + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice) if !fn(gf) { return @@ -486,6 +489,28 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) } } +func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) { + t := db.newReadOnlyTransaction() + defer t.close() + + dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil) + defer dbi.Release() + + var dk []byte + var gk []byte + var f FileIntf + var ok bool + for dbi.Next() { + gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate) + if !ok { + continue + } + if !fn(f) { + return + } + } +} + func (db *Instance) ListFolders() []string { t := db.newReadOnlyTransaction() defer t.close() @@ -511,36 +536,21 @@ func (db *Instance) ListFolders() []string { } func (db *Instance) dropFolder(folder []byte) { - t := db.newReadOnlyTransaction() + t := db.newReadWriteTransaction() defer t.close() - // Remove all items related to the given folder from the device->file bucket - dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) - for dbi.Next() { - itemFolder := db.deviceKeyFolder(dbi.Key()) - if bytes.Equal(folder, itemFolder) { - db.Delete(dbi.Key(), nil) - } + for _, key := range [][]byte{ + // Remove all items related to the given folder from the device->file bucket + db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen], + // Remove all sequences related to the folder + db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen], + // Remove all items related to the given folder from the global bucket + db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen], + // Remove all needs related to the folder + db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen], + } { + t.deleteKeyPrefix(key) } - dbi.Release() - - // Remove all sequences related to the folder - sequenceKey := db.sequenceKey([]byte(folder), 0) - dbi = t.NewIterator(util.BytesPrefix(sequenceKey[:keyPrefixLen+keyFolderLen]), nil) - for dbi.Next() { - db.Delete(dbi.Key(), nil) - } - dbi.Release() - - // Remove all items related to the given folder from the global bucket - dbi = t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil) - for dbi.Next() { - itemFolder, ok := db.globalKeyFolder(dbi.Key()) - if ok && bytes.Equal(folder, itemFolder) { - db.Delete(dbi.Key(), nil) - } - } - dbi.Release() } func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) { @@ -680,13 +690,14 @@ func (db *Instance) updateSchema0to1() { l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded) } +// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items +// to allow iteration in sequence order (simplifies sending indexes). func (db *Instance) updateSchema1to2() { t := db.newReadWriteTransaction() defer t.close() var sk []byte var dk []byte - for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { @@ -699,6 +710,34 @@ func (db *Instance) updateSchema1to2() { } } +// updateSchema2to3 introduces a needKey->nil bucket for locally needed files. +func (db *Instance) updateSchema2to3() { + t := db.newReadWriteTransaction() + defer t.close() + + var nk []byte + var dk []byte + for _, folderStr := range db.ListFolders() { + folder := []byte(folderStr) + db.withGlobal(folder, nil, true, func(f FileIntf) bool { + name := []byte(f.FileName()) + dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name) + var v protocol.Vector + haveFile, ok := db.getFileTrunc(dk, true) + if ok { + v = haveFile.FileVersion() + } + if !need(f, ok, v) { + return true + } + nk = t.db.needKeyInto(nk, folder, []byte(f.FileName())) + t.Put(nk, nil) + t.checkFlush() + return true + }) + } +} + // deviceKey returns a byte slice encoding the following information: // keyTypeDevice (1 byte) // folder (4 bytes) @@ -755,7 +794,7 @@ func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte { gk[0] = KeyTypeGlobal binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder)) copy(gk[keyPrefixLen+keyFolderLen:], file) - return gk[:reqLen] + return gk } // globalKeyName returns the filename from the key @@ -768,6 +807,17 @@ func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) { return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) } +// needKey is a globalKey with a different prefix +func (db *Instance) needKey(folder, file []byte) []byte { + return db.needKeyInto(nil, folder, file) +} + +func (db *Instance) needKeyInto(k, folder, file []byte) []byte { + k = db.globalKeyInto(k, folder, file) + k[0] = KeyTypeNeed + return k +} + // sequenceKey returns a byte slice encoding the following information: // KeyTypeSequence (1 byte) // folder (4 bytes) @@ -782,7 +832,7 @@ func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte { k[0] = KeyTypeSequence binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq)) - return k[:reqLen] + return k } // sequenceKeySequence returns the sequence number from the key diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 3b953b03..e724271c 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -224,3 +224,99 @@ func TestInvalidFiles(t *testing.T) { t.Error("quux should not be invalid") } } + +const myID = 1 + +var ( + remoteDevice0, remoteDevice1 protocol.DeviceID + update0to3Folder = "UpdateSchema0to3" + invalid = "invalid" + slashPrefixed = "/notgood" + haveUpdate0to3 map[protocol.DeviceID]fileList +) + +func init() { + remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") + remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU") + haveUpdate0to3 = map[protocol.DeviceID]fileList{ + protocol.LocalDeviceID: { + protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, + protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, + }, + remoteDevice0: { + protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, + protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true}, + protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)}, + }, + remoteDevice1: { + protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)}, + protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true}, + protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true}, + }, + } +} + +func TestUpdate0to3(t *testing.T) { + ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons") + + if err != nil { + t.Fatal(err) + } + db := newDBInstance(ldb, "") + + folder := []byte(update0to3Folder) + + db.updateSchema0to1() + + if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok { + t.Error("File prefixed by '/' was not removed during transition to schema 1") + } + + if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil { + t.Error("Invalid file wasn't added to global list") + } + + db.updateSchema1to2() + + found := false + db.withHaveSequence(folder, 0, func(fi FileIntf) bool { + f := fi.(protocol.FileInfo) + l.Infoln(f) + if found { + t.Error("Unexpected additional file via sequence", f.FileName()) + return true + } + if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) { + found = true + } else { + t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) + } + return true + }) + if !found { + t.Error("Local file wasn't added to sequence bucket", err) + } + + db.updateSchema2to3() + + need := map[string]protocol.FileInfo{ + haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0], + haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0], + haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2], + } + db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool { + e, ok := need[fi.FileName()] + if !ok { + t.Error("Got unexpected needed file:", fi.FileName()) + } + f := fi.(protocol.FileInfo) + delete(need, f.Name) + if !f.IsEquivalent(e, true, true) { + t.Errorf("Wrong needed file, got %v, expected %v", f, e) + } + return true + }) + for n := range need { + t.Errorf(`Missing needed file "%v"`, n) + } +} diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index 48f07638..f9c94917 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -12,6 +12,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" ) // A readOnlyTransaction represents a database snapshot. @@ -85,115 +86,87 @@ func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protoco // If the file does not have an entry in the global list, it is created. func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) bool { l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.Invalid) - name := []byte(file.Name) - svl, _ := t.Get(gk, nil) // skip error, we check len(svl) != 0 later var fl VersionList - var oldFile protocol.FileInfo - var hasOldFile bool - // Remove the device from the current version list - if len(svl) != 0 { - fl.Unmarshal(svl) // skip error, range handles success case - for i := range fl.Versions { - if bytes.Equal(fl.Versions[i].Device, device) { - if fl.Versions[i].Version.Equal(file.Version) && fl.Versions[i].Invalid == file.Invalid { - // No need to do anything - return false - } - - if i == 0 { - // Keep the current newest file around so we can subtract it from - // the metadata if we replace it. - oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name) - } - - fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...) - break - } - } + if svl, err := t.Get(gk, nil); err == nil { + fl.Unmarshal(svl) // Ignore error, continue with empty fl } - - nv := FileVersion{ - Device: device, - Version: file.Version, - Invalid: file.Invalid, - } - - insertedAt := -1 - // Find a position in the list to insert this file. The file at the front - // of the list is the newer, the "global". -insert: - for i := range fl.Versions { - switch fl.Versions[i].Version.Compare(file.Version) { - case protocol.Equal: - if nv.Invalid { - continue insert - } - fallthrough - - case protocol.Lesser: - // The version at this point in the list is equal to or lesser - // ("older") than us. We insert ourselves in front of it. - fl.Versions = insertVersion(fl.Versions, i, nv) - insertedAt = i - break insert - - case protocol.ConcurrentLesser, protocol.ConcurrentGreater: - // The version at this point is in conflict with us. We must pull - // the actual file metadata to determine who wins. If we win, we - // insert ourselves in front of the loser here. (The "Lesser" and - // "Greater" in the condition above is just based on the device - // IDs in the version vector, which is not the only thing we use - // to determine the winner.) - // - // A surprise missing file entry here is counted as a win for us. - if of, ok := t.getFile(folder, fl.Versions[i].Device, name); !ok || file.WinsConflict(of) { - fl.Versions = insertVersion(fl.Versions, i, nv) - insertedAt = i - break insert - } - } - } - + fl, removedFV, removedAt, insertedAt := fl.update(folder, device, file, t.db) if insertedAt == -1 { - // We didn't find a position for an insert above, so append to the end. - fl.Versions = append(fl.Versions, nv) - insertedAt = len(fl.Versions) - 1 - } - // Fixup the global size calculation. - if hasOldFile { - // We removed the previous newest version - meta.removeFile(globalDeviceID, oldFile) - if insertedAt == 0 { - // inserted a new newest version - meta.addFile(globalDeviceID, file) - } else { - // The previous second version is now the first - if newGlobal, ok := t.getFile(folder, fl.Versions[0].Device, name); ok { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - meta.addFile(globalDeviceID, newGlobal) - } - } - } else if insertedAt == 0 { - // We just inserted a new newest version. - meta.addFile(globalDeviceID, file) - if len(fl.Versions) > 1 { - // The previous newest version is now at index 1, grab it from there. - if oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name); ok { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - meta.removeFile(globalDeviceID, oldFile) - } - } + l.Debugln("update global; same version, global unchanged") + return false } - l.Debugf("new global after update: %v", fl) + if removedAt != 0 && insertedAt != 0 { + l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) + t.Put(gk, mustMarshal(&fl)) + return true + } + + name := []byte(file.Name) + + // Remove the old global from the global size counter + var oldGlobalFV FileVersion + if removedAt == 0 { + oldGlobalFV = removedFV + } else if len(fl.Versions) > 1 { + // The previous newest version is now at index 1 + oldGlobalFV = fl.Versions[1] + } + if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok { + // A failure to get the file here is surprising and our + // global size data will be incorrect until a restart... + meta.removeFile(globalDeviceID, oldFile) + } + + // Add the new global to the global size counter + var newGlobal protocol.FileInfo + if insertedAt == 0 { + // Inserted a new newest version + newGlobal = file + } else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok { + // The previous second version is now the first + newGlobal = new + } else { + panic("This file must exist in the db") + } + meta.addFile(globalDeviceID, newGlobal) + + // Fixup the list of files we need. + nk := t.db.needKey(folder, name) + hasNeeded, _ := t.db.Has(nk, nil) + if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) { + if !hasNeeded { + l.Debugf("local need insert; folder=%q, name=%q", folder, name) + t.Put(nk, nil) + } + } else if hasNeeded { + l.Debugf("local need delete; folder=%q, name=%q", folder, name) + t.Delete(nk) + } + + l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) t.Put(gk, mustMarshal(&fl)) return true } +func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool { + // We never need an invalid file. + if global.IsInvalid() { + return false + } + // We don't need a deleted file if we don't have it. + if global.IsDeleted() && !haveLocal { + return false + } + // We don't need the global file if we already have the same version. + if haveLocal && localVersion.Equal(global.FileVersion()) { + return false + } + return true +} + // removeFromGlobal removes the device from the global version list for the // given file. If the version list is empty after this, the file entry is // removed entirely. @@ -246,11 +219,13 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, } } -func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion { - t := append(vl, FileVersion{}) - copy(t[i+1:], t[i:]) - t[i] = v - return t +func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) { + dbi := t.NewIterator(util.BytesPrefix(prefix), nil) + for dbi.Next() { + t.Delete(dbi.Key()) + t.checkFlush() + } + dbi.Release() } type marshaller interface { diff --git a/lib/db/set.go b/lib/db/set.go index b6e55707..cdf155ed 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -44,6 +44,7 @@ type FileIntf interface { HasPermissionBits() bool SequenceNo() int64 BlockSize() int + FileVersion() protocol.Vector } // The Iterator is called with either a protocol.FileInfo or a diff --git a/lib/db/structs.go b/lib/db/structs.go index 60ff3582..d0c4061d 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -75,6 +75,10 @@ func (f FileInfoTruncated) SequenceNo() int64 { return f.Sequence } +func (f FileInfoTruncated) FileVersion() protocol.Vector { + return f.Version +} + func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo { return protocol.FileInfo{ Name: f.Name, diff --git a/lib/db/testdata/v0.14.45-update0to3.db.jsons b/lib/db/testdata/v0.14.45-update0to3.db.jsons new file mode 100644 index 00000000..87ef5241 --- /dev/null +++ b/lib/db/testdata/v0.14.45-update0to3.db.jsons @@ -0,0 +1,22 @@ +{"k":"AAAAAAAAAAABL25vdGdvb2Q=","v":"Cggvbm90Z29vZEoHCgUIARDoB1ACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHw=="} +{"k":"AAAAAAAAAAABYQ==","v":"CgFhSgcKBQgBEOgHUAGCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f"} +{"k":"AAAAAAAAAAACYg==","v":"CgFiSgcKBQgBEOkHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIA=="} +{"k":"AAAAAAAAAAACYw==","v":"CgFjOAFKBwoFCAEQ6geCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AAAAAAAAAAACZA==","v":"CgFkSgcKBQgBEOsHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="} +{"k":"AAAAAAAAAAADYw==","v":"CgFjSgcKBQgBEOoHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="} +{"k":"AAAAAAAAAAADZA==","v":"CgFkOAFKBwoFCAEQ6weCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AAAAAAAAAAADaW52YWxpZA==","v":"CgdpbnZhbGlkOAFKBwoFCAEQ7AeCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AQAAAAAvbm90Z29vZA==","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"} +{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"} +{"k":"AQAAAABi","v":"CisKBwoFCAEQ6QcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/At"} +{"k":"AQAAAABj","v":"CisKBwoFCAEQ6gcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcCi0KBwoFCAEQ6gcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtGAE="} +{"k":"AQAAAABk","v":"CisKBwoFCAEQ6wcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCi0KBwoFCAEQ6wcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="} +{"k":"AQAAAABpbnZhbGlk","v":"Ci0KBwoFCAEQ7AcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="} +{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHy9ub3Rnb29k","v":"AAAAAA=="} +{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2E=","v":"AAAAAA=="} +{"k":"BgAAAAAAAAAA","v":"VXBkYXRlU2NoZW1hMHRvMw=="} +{"k":"BwAAAAAAAAAA","v":""} +{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="} +{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="} +{"k":"BwAAAAMAAAAA","v":"R5QD/970lrX15bycChUiHnAHMWRQn6MHYa9jCU9vlFw="} +{"k":"CQAAAAA=","v":"CicIAjACigEg//////////////////////////////////////////8KJwgFMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAKKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLQolCAGKASBHlAP/3vSWtfXlvJwKFSIecAcxZFCfowdhr2MJT2+UXBCyn6iaw4HGlxU="} diff --git a/lib/db/util_test.go b/lib/db/util_test.go index 8eb525ac..a41de345 100644 --- a/lib/db/util_test.go +++ b/lib/db/util_test.go @@ -11,8 +11,6 @@ import ( "io" "os" - "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/util" @@ -58,37 +56,51 @@ func openJSONS(file string) (*leveldb.DB, error) { return db, nil } -func generateIgnoredFilesDB() { - // This generates a database with files with invalid flags, local and - // remote, in the format used in 0.14.48. +// The following commented tests were used to generate jsons files to stdout for +// future tests and are kept here for reference (reuse). - db := OpenMemory() - fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) - fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ - { // invalid (ignored) file - Name: "foo", - Type: protocol.FileInfoTypeFile, - Invalid: true, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, - }, - { // regular file - Name: "bar", - Type: protocol.FileInfoTypeFile, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, - }, - }) - fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ - { // invalid file - Name: "baz", - Type: protocol.FileInfoTypeFile, - Invalid: true, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, - }, - { // regular file - Name: "quux", - Type: protocol.FileInfoTypeFile, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, - }, - }) - writeJSONS(os.Stdout, db.DB) -} +// TestGenerateIgnoredFilesDB generates a database with files with invalid flags, +// local and remote, in the format used in 0.14.48. +// func TestGenerateIgnoredFilesDB(t *testing.T) { +// db := OpenMemory() +// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) +// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ +// { // invalid (ignored) file +// Name: "foo", +// Type: protocol.FileInfoTypeFile, +// Invalid: true, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, +// }, +// { // regular file +// Name: "bar", +// Type: protocol.FileInfoTypeFile, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, +// }, +// }) +// fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ +// { // invalid file +// Name: "baz", +// Type: protocol.FileInfoTypeFile, +// Invalid: true, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, +// }, +// { // regular file +// Name: "quux", +// Type: protocol.FileInfoTypeFile, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, +// }, +// }) +// writeJSONS(os.Stdout, db.DB) +// } + +// TestGenerateUpdate0to3DB generates a database with files with invalid flags, prefixed +// by a slash and other files to test database migration from version 0 to 3, in the +// format used in 0.14.45. +// func TestGenerateUpdate0to3DB(t *testing.T) { +// db := OpenMemory() +// fs := NewFileSet(update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) +// for devID, files := range haveUpdate0to3 { +// fs.Update(devID, files) +// } +// writeJSONS(os.Stdout, db.DB) +// } diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index 4c1514da..19261dd5 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -346,7 +346,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) { expected := map[string]struct{}{invIgn: {}, ign: {}, ignExisting: {}} for _, f := range fs { if _, ok := expected[f.Name]; !ok { - t.Fatalf("Unexpected file %v was added to index", f.Name) + t.Errorf("Unexpected file %v was added to index", f.Name) } if !f.Invalid { t.Errorf("File %v wasn't marked as invalid", f.Name) diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 5dde3ea9..cfc4b988 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -95,6 +95,10 @@ func (f FileInfo) SequenceNo() int64 { return f.Sequence } +func (f FileInfo) FileVersion() Vector { + return f.Version +} + // WinsConflict returns true if "f" is the one to choose when it is in // conflict with "other". func (f FileInfo) WinsConflict(other FileInfo) bool {