diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 67e937ef..9f4eed6e 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -751,9 +751,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) { miscDB.PutString("prevVersion", Version) } - // Potential database transitions - ldb.UpdateSchema() - m := model.NewModel(cfg, myID, "syncthing", Version, ldb, protectedFiles) if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" { diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index 2b82b23d..728d5533 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -13,8 +13,6 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -const dbVersion = 3 - const ( KeyTypeDevice = iota KeyTypeGlobal diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index 19dadbcf..0d43439e 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -83,34 +83,10 @@ func newDBInstance(db *leveldb.DB, location string) *Instance { } i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx}) i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx}) + i.updateSchema() return i } -// UpdateSchema does transitions to the current db version if necessary -func (db *Instance) UpdateSchema() { - miscDB := NewNamespacedKV(db, string(KeyTypeMiscData)) - prevVersion, _ := miscDB.Int64("dbVersion") - - if prevVersion >= dbVersion { - return - } - - l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion) - - if prevVersion == 0 { - db.updateSchema0to1() - } - if prevVersion <= 1 { - db.updateSchema1to2() - } - if prevVersion <= 2 { - db.updateSchema2to3() - } - - l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion) - miscDB.PutInt64("dbVersion", dbVersion) -} - // Committed returns the number of items committed to the database since startup func (db *Instance) Committed() int64 { return atomic.LoadInt64(&db.committed) @@ -620,124 +596,6 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { l.Debugf("db check completed for %q", folder) } -func (db *Instance) updateSchema0to1() { - t := db.newReadWriteTransaction() - defer t.close() - - dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) - defer dbi.Release() - - symlinkConv := 0 - changedFolders := make(map[string]struct{}) - ignAdded := 0 - meta := newMetadataTracker() // dummy metadata tracker - var gk []byte - - for dbi.Next() { - folder := db.deviceKeyFolder(dbi.Key()) - device := db.deviceKeyDevice(dbi.Key()) - name := db.deviceKeyName(dbi.Key()) - - // Remove files with absolute path (see #4799) - if strings.HasPrefix(string(name), "/") { - if _, ok := changedFolders[string(folder)]; !ok { - changedFolders[string(folder)] = struct{}{} - } - gk = db.globalKeyInto(gk, folder, name) - t.removeFromGlobal(gk, folder, device, nil, nil) - t.Delete(dbi.Key()) - t.checkFlush() - continue - } - - // Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK - // type (previously SYMLINK_UNKNOWN). It does this for all devices, both - // local and remote, and does not reset delta indexes. It shouldn't really - // matter what the symlink type is, but this cleans it up for a possible - // future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood. - var f protocol.FileInfo - if err := f.Unmarshal(dbi.Value()); err != nil { - // probably can't happen - continue - } - if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile { - f.Type = protocol.FileInfoTypeSymlink - bs, err := f.Marshal() - if err != nil { - panic("can't happen: " + err.Error()) - } - t.Put(dbi.Key(), bs) - t.checkFlush() - symlinkConv++ - } - - // Add invalid files to global list - if f.Invalid { - gk = db.globalKeyInto(gk, folder, name) - if t.updateGlobal(gk, folder, device, f, meta) { - if _, ok := changedFolders[string(folder)]; !ok { - changedFolders[string(folder)] = struct{}{} - } - ignAdded++ - } - } - } - - for folder := range changedFolders { - db.dropFolderMeta([]byte(folder)) - } - - l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded) -} - -// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items -// to allow iteration in sequence order (simplifies sending indexes). -func (db *Instance) updateSchema1to2() { - t := db.newReadWriteTransaction() - defer t.close() - - var sk []byte - var dk []byte - for _, folderStr := range db.ListFolders() { - folder := []byte(folderStr) - db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { - sk = db.sequenceKeyInto(sk, folder, f.SequenceNo()) - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName())) - t.Put(sk, dk) - t.checkFlush() - return true - }) - } -} - -// updateSchema2to3 introduces a needKey->nil bucket for locally needed files. -func (db *Instance) updateSchema2to3() { - t := db.newReadWriteTransaction() - defer t.close() - - var nk []byte - var dk []byte - for _, folderStr := range db.ListFolders() { - folder := []byte(folderStr) - db.withGlobal(folder, nil, true, func(f FileIntf) bool { - name := []byte(f.FileName()) - dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name) - var v protocol.Vector - haveFile, ok := db.getFileTrunc(dk, true) - if ok { - v = haveFile.FileVersion() - } - if !need(f, ok, v) { - return true - } - nk = t.db.needKeyInto(nk, folder, []byte(f.FileName())) - t.Put(nk, nil) - t.checkFlush() - return true - }) - } -} - // deviceKey returns a byte slice encoding the following information: // keyTypeDevice (1 byte) // folder (4 bytes) diff --git a/lib/db/leveldb_dbinstance_updateschema.go b/lib/db/leveldb_dbinstance_updateschema.go new file mode 100644 index 00000000..f65df598 --- /dev/null +++ b/lib/db/leveldb_dbinstance_updateschema.go @@ -0,0 +1,155 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "strings" + + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syndtr/goleveldb/leveldb/util" +) + +const dbVersion = 3 + +func (db *Instance) updateSchema() { + miscDB := NewNamespacedKV(db, string(KeyTypeMiscData)) + prevVersion, _ := miscDB.Int64("dbVersion") + + if prevVersion >= dbVersion { + return + } + + l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion) + + if prevVersion < 1 { + db.updateSchema0to1() + } + if prevVersion < 2 { + db.updateSchema1to2() + } + if prevVersion < 3 { + db.updateSchema2to3() + } + + miscDB.PutInt64("dbVersion", dbVersion) +} + +func (db *Instance) updateSchema0to1() { + t := db.newReadWriteTransaction() + defer t.close() + + dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) + defer dbi.Release() + + symlinkConv := 0 + changedFolders := make(map[string]struct{}) + ignAdded := 0 + meta := newMetadataTracker() // dummy metadata tracker + var gk []byte + + for dbi.Next() { + folder := db.deviceKeyFolder(dbi.Key()) + device := db.deviceKeyDevice(dbi.Key()) + name := db.deviceKeyName(dbi.Key()) + + // Remove files with absolute path (see #4799) + if strings.HasPrefix(string(name), "/") { + if _, ok := changedFolders[string(folder)]; !ok { + changedFolders[string(folder)] = struct{}{} + } + gk = db.globalKeyInto(gk, folder, name) + t.removeFromGlobal(gk, folder, device, nil, nil) + t.Delete(dbi.Key()) + t.checkFlush() + continue + } + + // Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK + // type (previously SYMLINK_UNKNOWN). It does this for all devices, both + // local and remote, and does not reset delta indexes. It shouldn't really + // matter what the symlink type is, but this cleans it up for a possible + // future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood. + var f protocol.FileInfo + if err := f.Unmarshal(dbi.Value()); err != nil { + // probably can't happen + continue + } + if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile { + f.Type = protocol.FileInfoTypeSymlink + bs, err := f.Marshal() + if err != nil { + panic("can't happen: " + err.Error()) + } + t.Put(dbi.Key(), bs) + t.checkFlush() + symlinkConv++ + } + + // Add invalid files to global list + if f.Invalid { + gk = db.globalKeyInto(gk, folder, name) + if t.updateGlobal(gk, folder, device, f, meta) { + if _, ok := changedFolders[string(folder)]; !ok { + changedFolders[string(folder)] = struct{}{} + } + ignAdded++ + } + } + } + + for folder := range changedFolders { + db.dropFolderMeta([]byte(folder)) + } +} + +// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items +// to allow iteration in sequence order (simplifies sending indexes). +func (db *Instance) updateSchema1to2() { + t := db.newReadWriteTransaction() + defer t.close() + + var sk []byte + var dk []byte + for _, folderStr := range db.ListFolders() { + folder := []byte(folderStr) + db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { + sk = db.sequenceKeyInto(sk, folder, f.SequenceNo()) + dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName())) + t.Put(sk, dk) + t.checkFlush() + return true + }) + } +} + +// updateSchema2to3 introduces a needKey->nil bucket for locally needed files. +func (db *Instance) updateSchema2to3() { + t := db.newReadWriteTransaction() + defer t.close() + + var nk []byte + var dk []byte + for _, folderStr := range db.ListFolders() { + folder := []byte(folderStr) + db.withGlobal(folder, nil, true, func(f FileIntf) bool { + name := []byte(f.FileName()) + dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name) + var v protocol.Vector + haveFile, ok := db.getFileTrunc(dk, true) + if ok { + v = haveFile.FileVersion() + } + if !need(f, ok, v) { + return true + } + nk = t.db.needKeyInto(nk, folder, []byte(f.FileName())) + t.Put(nk, nil) + t.checkFlush() + return true + }) + } +}