From 2a4fc28318767fe7aa10fcbe7b7fd86c5cd9a2a7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 31 Oct 2015 12:31:25 +0100 Subject: [PATCH] We should pass around db.Instance instead of leveldb.DB We're going to need the db.Instance to keep some state, and for that to work we need the same one passed around everywhere. Hence this moves the leveldb-specific file opening stuff into the db package and exports the dbInstance type. --- cmd/syncthing/main.go | 74 ++------------------- cmd/syncthing/main_test.go | 5 +- lib/config/config_test.go | 2 - lib/config/optionsconfiguration.go | 1 - lib/db/blockmap.go | 8 +-- lib/db/blockmap_test.go | 12 +--- lib/db/leveldb_dbinstance.go | 101 ++++++++++++++++++++++------- lib/db/leveldb_test.go | 4 +- lib/db/leveldb_transactions.go | 6 +- lib/db/namespaced.go | 4 +- lib/db/namespaced_test.go | 23 ++----- lib/db/set.go | 18 ++--- lib/db/set_test.go | 57 ++++------------ lib/db/virtualmtime.go | 4 +- lib/db/virtualmtime_test.go | 8 +-- lib/model/model.go | 5 +- lib/model/model_test.go | 30 ++++----- lib/model/rwfolder_test.go | 18 +++-- lib/stats/device.go | 3 +- lib/stats/folder.go | 3 +- 20 files changed, 145 insertions(+), 241 deletions(-) diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 2d232a6b..749d21e0 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -9,6 +9,7 @@ package main import ( "bytes" "crypto/tls" + "errors" "flag" "fmt" "io/ioutil" @@ -42,9 +43,6 @@ import ( "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/syncthing/lib/upgrade" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" "github.com/thejerf/suture" ) @@ -371,7 +369,7 @@ func main() { if doUpgrade { // Use leveldb database locks to protect against concurrent upgrades - _, err = leveldb.OpenFile(locations[locDatabase], &opt.Options{OpenFilesCacheCapacity: 100}) + _, err = db.Open(locations[locDatabase]) if err != nil { l.Infoln("Attempting upgrade through running Syncthing...") err = upgradeViaRest() @@ -617,21 +615,7 @@ func syncthingMain() { } dbFile := locations[locDatabase] - dbOpts := dbOpts(cfg) - ldb, err := leveldb.OpenFile(dbFile, dbOpts) - if leveldbIsCorrupted(err) { - ldb, err = leveldb.RecoverFile(dbFile, dbOpts) - } - if leveldbIsCorrupted(err) { - // The database is corrupted, and we've tried to recover it but it - // didn't work. At this point there isn't much to do beyond dropping - // the database and reindexing... - l.Infoln("Database corruption detected, unable to recover. Reinitializing...") - if err := resetDB(); err != nil { - l.Fatalln("Remove database:", err) - } - ldb, err = leveldb.OpenFile(dbFile, dbOpts) - } + ldb, err := db.Open(dbFile) if err != nil { l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?") } @@ -642,7 +626,7 @@ func syncthingMain() { // Remove database entries for folders that no longer exist in the config folders := cfg.Folders() - for _, folder := range db.ListFolders(ldb) { + for _, folder := range ldb.ListFolders() { if _, ok := folders[folder]; !ok { l.Infof("Cleaning data for dropped folder %q", folder) db.DropFolder(ldb, folder) @@ -881,40 +865,6 @@ func loadConfig(cfgFile string) (*config.Wrapper, string, error) { return cfg, myName, nil } -func dbOpts(cfg *config.Wrapper) *opt.Options { - // Calculate a suitable database block cache capacity. - - // Default is 8 MiB. - blockCacheCapacity := 8 << 20 - // Increase block cache up to this maximum: - const maxCapacity = 64 << 20 - // ... which we reach when the box has this much RAM: - const maxAtRAM = 8 << 30 - - if v := cfg.Options().DatabaseBlockCacheMiB; v != 0 { - // Use the value from the config, if it's set. - blockCacheCapacity = v << 20 - } else if bytes, err := memorySize(); err == nil { - // We start at the default of 8 MiB and use larger values for machines - // with more memory. - - if bytes > maxAtRAM { - // Cap the cache at maxCapacity when we reach maxAtRam amount of memory - blockCacheCapacity = maxCapacity - } else if bytes > maxAtRAM/maxCapacity*int64(blockCacheCapacity) { - // Grow from the default to maxCapacity at maxAtRam amount of memory - blockCacheCapacity = int(bytes * maxCapacity / maxAtRAM) - } - l.Infoln("Database block cache capacity", blockCacheCapacity/1024, "KiB") - } - - return &opt.Options{ - OpenFilesCacheCapacity: 100, - BlockCacheCapacity: blockCacheCapacity, - WriteBuffer: 4 << 20, - } -} - func startAuditing(mainSvc *suture.Supervisor) { auditFile := timestampedLoc(locAuditLog) fd, err := os.OpenFile(auditFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) @@ -1166,19 +1116,3 @@ func checkShortIDs(cfg *config.Wrapper) error { } return nil } - -// A "better" version of leveldb's errors.IsCorrupted. -func leveldbIsCorrupted(err error) bool { - switch { - case err == nil: - return false - - case errors.IsCorrupted(err): - return true - - case strings.Contains(err.Error(), "corrupted"): - return true - } - - return false -} diff --git a/cmd/syncthing/main_test.go b/cmd/syncthing/main_test.go index fd11f109..b42b4be1 100644 --- a/cmd/syncthing/main_test.go +++ b/cmd/syncthing/main_test.go @@ -14,9 +14,6 @@ import ( "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) func TestFolderErrors(t *testing.T) { @@ -38,7 +35,7 @@ func TestFolderErrors(t *testing.T) { } } - ldb, _ := leveldb.Open(storage.NewMemStorage(), nil) + ldb := db.OpenMemory() // Case 1 - new folder, directory and marker created diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 28da9b91..0f95524e 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -56,7 +56,6 @@ func TestDefaultValues(t *testing.T) { ProgressUpdateIntervalS: 5, SymlinksEnabled: true, LimitBandwidthInLan: false, - DatabaseBlockCacheMiB: 0, MinHomeDiskFreePct: 1, URURL: "https://data.syncthing.net/newdata", URInitialDelayS: 1800, @@ -180,7 +179,6 @@ func TestOverriddenValues(t *testing.T) { ProgressUpdateIntervalS: 10, SymlinksEnabled: false, LimitBandwidthInLan: true, - DatabaseBlockCacheMiB: 42, MinHomeDiskFreePct: 5.2, URURL: "https://localhost/newdata", URInitialDelayS: 800, diff --git a/lib/config/optionsconfiguration.go b/lib/config/optionsconfiguration.go index 1a992634..b0a25525 100644 --- a/lib/config/optionsconfiguration.go +++ b/lib/config/optionsconfiguration.go @@ -37,7 +37,6 @@ type OptionsConfiguration struct { ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"` SymlinksEnabled bool `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"` LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"` - DatabaseBlockCacheMiB int `xml:"databaseBlockCacheMiB" json:"databaseBlockCacheMiB" default:"0"` MinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"` ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=30"` AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"` diff --git a/lib/db/blockmap.go b/lib/db/blockmap.go index a4b8abc5..ec932568 100644 --- a/lib/db/blockmap.go +++ b/lib/db/blockmap.go @@ -29,11 +29,11 @@ var blockFinder *BlockFinder const maxBatchSize = 256 << 10 type BlockMap struct { - db *leveldb.DB + db *Instance folder string } -func NewBlockMap(db *leveldb.DB, folder string) *BlockMap { +func NewBlockMap(db *Instance, folder string) *BlockMap { return &BlockMap{ db: db, folder: folder, @@ -146,10 +146,10 @@ func (m *BlockMap) blockKeyInto(o, hash []byte, file string) []byte { } type BlockFinder struct { - db *leveldb.DB + db *Instance } -func NewBlockFinder(db *leveldb.DB) *BlockFinder { +func NewBlockFinder(db *Instance) *BlockFinder { if blockFinder != nil { return blockFinder } diff --git a/lib/db/blockmap_test.go b/lib/db/blockmap_test.go index 110383d2..a3317c6d 100644 --- a/lib/db/blockmap_test.go +++ b/lib/db/blockmap_test.go @@ -10,9 +10,6 @@ import ( "testing" "github.com/syncthing/syncthing/lib/protocol" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) func genBlocks(n int) []protocol.BlockInfo { @@ -50,17 +47,14 @@ func init() { } } -func setup() (*leveldb.DB, *BlockFinder) { +func setup() (*Instance, *BlockFinder) { // Setup - db, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - panic(err) - } + db := OpenMemory() return db, NewBlockFinder(db) } -func dbEmpty(db *leveldb.DB) bool { +func dbEmpty(db *Instance) bool { iter := db.NewIterator(nil, nil) defer iter.Release() if iter.Next() { diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index b5acb42a..f7996497 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -8,27 +8,64 @@ package db import ( "bytes" + "os" "sort" + "strings" "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/util" ) type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64 -type dbInstance struct { +type Instance struct { *leveldb.DB } -func newDBInstance(db *leveldb.DB) *dbInstance { - return &dbInstance{ +func Open(file string) (*Instance, error) { + opts := &opt.Options{ + OpenFilesCacheCapacity: 100, + WriteBuffer: 4 << 20, + } + + db, err := leveldb.OpenFile(file, opts) + if leveldbIsCorrupted(err) { + db, err = leveldb.RecoverFile(file, opts) + } + if leveldbIsCorrupted(err) { + // The database is corrupted, and we've tried to recover it but it + // didn't work. At this point there isn't much to do beyond dropping + // the database and reindexing... + l.Infoln("Database corruption detected, unable to recover. Reinitializing...") + if err := os.RemoveAll(file); err != nil { + return nil, err + } + db, err = leveldb.OpenFile(file, opts) + } + if err != nil { + return nil, err + } + + return newDBInstance(db), nil +} + +func OpenMemory() *Instance { + db, _ := leveldb.Open(storage.NewMemStorage(), nil) + return newDBInstance(db) +} + +func newDBInstance(db *leveldb.DB) *Instance { + return &Instance{ DB: db, } } -func (db *dbInstance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 { +func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 { sort.Sort(fileList(fs)) // sort list on name, same as in the database start := db.deviceKey(folder, device, nil) // before all folder/device files @@ -126,7 +163,7 @@ func (db *dbInstance) genericReplace(folder, device []byte, fs []protocol.FileIn return maxLocalVer } -func (db *dbInstance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 { +func (db *Instance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 { // TODO: Return the remaining maxLocalVer? return db.genericReplace(folder, device, fs, localSize, globalSize, func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64 { // Database has a file that we are missing. Remove it. @@ -137,7 +174,7 @@ func (db *dbInstance) replace(folder, device []byte, fs []protocol.FileInfo, loc }) } -func (db *dbInstance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 { +func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 { t := db.newReadWriteTransaction() defer t.close() @@ -195,7 +232,7 @@ func (db *dbInstance) updateFiles(folder, device []byte, fs []protocol.FileInfo, return maxLocalVer } -func (db *dbInstance) withHave(folder, device []byte, truncate bool, fn Iterator) { +func (db *Instance) withHave(folder, device []byte, truncate bool, fn Iterator) { start := db.deviceKey(folder, device, nil) // before all folder/device files limit := db.deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files @@ -216,7 +253,7 @@ func (db *dbInstance) withHave(folder, device []byte, truncate bool, fn Iterator } } -func (db *dbInstance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) { +func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) { start := db.deviceKey(folder, nil, nil) // before all folder/device files limit := db.deviceKey(folder, protocol.LocalDeviceID[:], []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files @@ -249,11 +286,11 @@ func (db *dbInstance) withAllFolderTruncated(folder []byte, fn func(device []byt } } -func (db *dbInstance) getFile(folder, device, file []byte) (protocol.FileInfo, bool) { +func (db *Instance) getFile(folder, device, file []byte) (protocol.FileInfo, bool) { return getFile(db, db.deviceKey(folder, device, file)) } -func (db *dbInstance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) { +func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) { k := db.globalKey(folder, file) t := db.newReadOnlyTransaction() @@ -290,7 +327,7 @@ func (db *dbInstance) getGlobal(folder, file []byte, truncate bool) (FileIntf, b return fi, true } -func (db *dbInstance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) { +func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) { t := db.newReadOnlyTransaction() defer t.close() @@ -333,7 +370,7 @@ func (db *dbInstance) withGlobal(folder, prefix []byte, truncate bool, fn Iterat } } -func (db *dbInstance) availability(folder, file []byte) []protocol.DeviceID { +func (db *Instance) availability(folder, file []byte) []protocol.DeviceID { k := db.globalKey(folder, file) bs, err := db.Get(k, nil) if err == leveldb.ErrNotFound { @@ -361,7 +398,7 @@ func (db *dbInstance) availability(folder, file []byte) []protocol.DeviceID { return devices } -func (db *dbInstance) withNeed(folder, device []byte, truncate bool, fn Iterator) { +func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) { start := db.globalKey(folder, nil) limit := db.globalKey(folder, []byte{0xff, 0xff, 0xff, 0xff}) @@ -452,7 +489,7 @@ nextFile: } } -func (db *dbInstance) listFolders() []string { +func (db *Instance) ListFolders() []string { t := db.newReadOnlyTransaction() defer t.close() @@ -476,7 +513,7 @@ func (db *dbInstance) listFolders() []string { return folders } -func (db *dbInstance) dropFolder(folder []byte) { +func (db *Instance) dropFolder(folder []byte) { t := db.newReadOnlyTransaction() defer t.close() @@ -501,7 +538,7 @@ func (db *dbInstance) dropFolder(folder []byte) { dbi.Release() } -func (db *dbInstance) checkGlobals(folder []byte, globalSize *sizeTracker) { +func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) { t := db.newReadWriteTransaction() defer t.close() @@ -560,11 +597,11 @@ func (db *dbInstance) checkGlobals(folder []byte, globalSize *sizeTracker) { // folder (64 bytes) // device (32 bytes) // name (variable size) -func (db *dbInstance) deviceKey(folder, device, file []byte) []byte { +func (db *Instance) deviceKey(folder, device, file []byte) []byte { return db.deviceKeyInto(nil, folder, device, file) } -func (db *dbInstance) deviceKeyInto(k []byte, folder, device, file []byte) []byte { +func (db *Instance) deviceKeyInto(k []byte, folder, device, file []byte) []byte { reqLen := 1 + 64 + 32 + len(file) if len(k) < reqLen { k = make([]byte, reqLen) @@ -579,11 +616,11 @@ func (db *dbInstance) deviceKeyInto(k []byte, folder, device, file []byte) []byt return k[:reqLen] } -func (db *dbInstance) deviceKeyName(key []byte) []byte { +func (db *Instance) deviceKeyName(key []byte) []byte { return key[1+64+32:] } -func (db *dbInstance) deviceKeyFolder(key []byte) []byte { +func (db *Instance) deviceKeyFolder(key []byte) []byte { folder := key[1 : 1+64] izero := bytes.IndexByte(folder, 0) if izero < 0 { @@ -592,7 +629,7 @@ func (db *dbInstance) deviceKeyFolder(key []byte) []byte { return folder[:izero] } -func (db *dbInstance) deviceKeyDevice(key []byte) []byte { +func (db *Instance) deviceKeyDevice(key []byte) []byte { return key[1+64 : 1+64+32] } @@ -600,7 +637,7 @@ func (db *dbInstance) deviceKeyDevice(key []byte) []byte { // keyTypeGlobal (1 byte) // folder (64 bytes) // name (variable size) -func (db *dbInstance) globalKey(folder, file []byte) []byte { +func (db *Instance) globalKey(folder, file []byte) []byte { k := make([]byte, 1+64+len(file)) k[0] = KeyTypeGlobal if len(folder) > 64 { @@ -611,11 +648,11 @@ func (db *dbInstance) globalKey(folder, file []byte) []byte { return k } -func (db *dbInstance) globalKeyName(key []byte) []byte { +func (db *Instance) globalKeyName(key []byte) []byte { return key[1+64:] } -func (db *dbInstance) globalKeyFolder(key []byte) []byte { +func (db *Instance) globalKeyFolder(key []byte) []byte { folder := key[1 : 1+64] izero := bytes.IndexByte(folder, 0) if izero < 0 { @@ -635,3 +672,19 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) { err := tf.UnmarshalXDR(bs) return tf, err } + +// A "better" version of leveldb's errors.IsCorrupted. +func leveldbIsCorrupted(err error) bool { + switch { + case err == nil: + return false + + case errors.IsCorrupted(err): + return true + + case strings.Contains(err.Error(), "corrupted"): + return true + } + + return false +} diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 2bd020ed..823da1b9 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -16,7 +16,7 @@ func TestDeviceKey(t *testing.T) { dev := []byte("device67890123456789012345678901") name := []byte("name") - db := &dbInstance{} + db := &Instance{} key := db.deviceKey(fld, dev, name) @@ -38,7 +38,7 @@ func TestGlobalKey(t *testing.T) { fld := []byte("folder6789012345678901234567890123456789012345678901234567890123") name := []byte("name") - db := &dbInstance{} + db := &Instance{} key := db.globalKey(fld, name) diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index f4ab2cb4..2f9236a5 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -16,10 +16,10 @@ import ( // A readOnlyTransaction represents a database snapshot. type readOnlyTransaction struct { *leveldb.Snapshot - db *dbInstance + db *Instance } -func (db *dbInstance) newReadOnlyTransaction() readOnlyTransaction { +func (db *Instance) newReadOnlyTransaction() readOnlyTransaction { snap, err := db.GetSnapshot() if err != nil { panic(err) @@ -46,7 +46,7 @@ type readWriteTransaction struct { *leveldb.Batch } -func (db *dbInstance) newReadWriteTransaction() readWriteTransaction { +func (db *Instance) newReadWriteTransaction() readWriteTransaction { t := db.newReadOnlyTransaction() return readWriteTransaction{ readOnlyTransaction: t, diff --git a/lib/db/namespaced.go b/lib/db/namespaced.go index d1b1bf15..eb4103f9 100644 --- a/lib/db/namespaced.go +++ b/lib/db/namespaced.go @@ -17,13 +17,13 @@ import ( // NamespacedKV is a simple key-value store using a specific namespace within // a leveldb. type NamespacedKV struct { - db *leveldb.DB + db *Instance prefix []byte } // NewNamespacedKV returns a new NamespacedKV that lives in the namespace // specified by the prefix. -func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV { +func NewNamespacedKV(db *Instance, prefix string) *NamespacedKV { return &NamespacedKV{ db: db, prefix: []byte(prefix), diff --git a/lib/db/namespaced_test.go b/lib/db/namespaced_test.go index b55a21ef..3d0fdc41 100644 --- a/lib/db/namespaced_test.go +++ b/lib/db/namespaced_test.go @@ -9,16 +9,10 @@ package db import ( "testing" "time" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) func TestNamespacedInt(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := OpenMemory() n1 := NewNamespacedKV(ldb, "foo") n2 := NewNamespacedKV(ldb, "bar") @@ -53,10 +47,7 @@ func TestNamespacedInt(t *testing.T) { } func TestNamespacedTime(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := OpenMemory() n1 := NewNamespacedKV(ldb, "foo") @@ -73,10 +64,7 @@ func TestNamespacedTime(t *testing.T) { } func TestNamespacedString(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := OpenMemory() n1 := NewNamespacedKV(ldb, "foo") @@ -92,10 +80,7 @@ func TestNamespacedString(t *testing.T) { } func TestNamespacedReset(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := OpenMemory() n1 := NewNamespacedKV(ldb, "foo") diff --git a/lib/db/set.go b/lib/db/set.go index 4b41f0ec..481adbe6 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -18,14 +18,13 @@ import ( "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" - "github.com/syndtr/goleveldb/leveldb" ) type FileSet struct { localVersion map[protocol.DeviceID]int64 mutex sync.Mutex folder string - db *dbInstance + db *Instance blockmap *BlockMap localSize sizeTracker globalSize sizeTracker @@ -93,11 +92,11 @@ func (s *sizeTracker) Size() (files, deleted int, bytes int64) { return s.files, s.deleted, s.bytes } -func NewFileSet(folder string, db *leveldb.DB) *FileSet { +func NewFileSet(folder string, db *Instance) *FileSet { var s = FileSet{ localVersion: make(map[protocol.DeviceID]int64), folder: folder, - db: newDBInstance(db), + db: db, blockmap: NewBlockMap(db, folder), mutex: sync.NewMutex(), } @@ -239,17 +238,10 @@ func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) { return s.globalSize.Size() } -// ListFolders returns the folder IDs seen in the database. -func ListFolders(db *leveldb.DB) []string { - i := newDBInstance(db) - return i.listFolders() -} - // DropFolder clears out all information related to the given folder from the // database. -func DropFolder(db *leveldb.DB, folder string) { - i := newDBInstance(db) - i.dropFolder([]byte(folder)) +func DropFolder(db *Instance, folder string) { + db.dropFolder([]byte(folder)) bm := &BlockMap{ db: db, folder: folder, diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 00e83159..98aee3dd 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -15,8 +15,6 @@ import ( "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) var remoteDevice0, remoteDevice1 protocol.DeviceID @@ -96,11 +94,7 @@ func (l fileList) String() string { } func TestGlobalSet(t *testing.T) { - - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() m := db.NewFileSet("test", ldb) @@ -303,10 +297,7 @@ func TestGlobalSet(t *testing.T) { } func TestNeedWithInvalid(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s := db.NewFileSet("test", ldb) @@ -343,10 +334,7 @@ func TestNeedWithInvalid(t *testing.T) { } func TestUpdateToInvalid(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s := db.NewFileSet("test", ldb) @@ -378,10 +366,7 @@ func TestUpdateToInvalid(t *testing.T) { } func TestInvalidAvailability(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s := db.NewFileSet("test", ldb) @@ -419,10 +404,7 @@ func TestInvalidAvailability(t *testing.T) { } func TestGlobalReset(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() m := db.NewFileSet("test", ldb) @@ -460,10 +442,7 @@ func TestGlobalReset(t *testing.T) { } func TestNeed(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() m := db.NewFileSet("test", ldb) @@ -501,10 +480,7 @@ func TestNeed(t *testing.T) { } func TestLocalVersion(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() m := db.NewFileSet("test", ldb) @@ -534,10 +510,7 @@ func TestLocalVersion(t *testing.T) { } func TestListDropFolder(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s0 := db.NewFileSet("test0", ldb) local1 := []protocol.FileInfo{ @@ -558,7 +531,7 @@ func TestListDropFolder(t *testing.T) { // Check that we have both folders and their data is in the global list expectedFolderList := []string{"test0", "test1"} - if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) { + if actualFolderList := ldb.ListFolders(); !reflect.DeepEqual(actualFolderList, expectedFolderList) { t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList) } if l := len(globalList(s0)); l != 3 { @@ -573,7 +546,7 @@ func TestListDropFolder(t *testing.T) { db.DropFolder(ldb, "test1") expectedFolderList = []string{"test0"} - if actualFolderList := db.ListFolders(ldb); !reflect.DeepEqual(actualFolderList, expectedFolderList) { + if actualFolderList := ldb.ListFolders(); !reflect.DeepEqual(actualFolderList, expectedFolderList) { t.Fatalf("FolderList mismatch\nE: %v\nA: %v", expectedFolderList, actualFolderList) } if l := len(globalList(s0)); l != 3 { @@ -585,10 +558,7 @@ func TestListDropFolder(t *testing.T) { } func TestGlobalNeedWithInvalid(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s := db.NewFileSet("test1", ldb) @@ -625,10 +595,7 @@ func TestGlobalNeedWithInvalid(t *testing.T) { } func TestLongPath(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := db.OpenMemory() s := db.NewFileSet("test", ldb) diff --git a/lib/db/virtualmtime.go b/lib/db/virtualmtime.go index 9bc1936f..a632a786 100644 --- a/lib/db/virtualmtime.go +++ b/lib/db/virtualmtime.go @@ -9,8 +9,6 @@ package db import ( "fmt" "time" - - "github.com/syndtr/goleveldb/leveldb" ) // This type encapsulates a repository of mtimes for platforms where file mtimes @@ -25,7 +23,7 @@ type VirtualMtimeRepo struct { ns *NamespacedKV } -func NewVirtualMtimeRepo(ldb *leveldb.DB, folder string) *VirtualMtimeRepo { +func NewVirtualMtimeRepo(ldb *Instance, folder string) *VirtualMtimeRepo { prefix := string(KeyTypeVirtualMtime) + folder return &VirtualMtimeRepo{ diff --git a/lib/db/virtualmtime_test.go b/lib/db/virtualmtime_test.go index d3d7de46..48c863c7 100644 --- a/lib/db/virtualmtime_test.go +++ b/lib/db/virtualmtime_test.go @@ -9,16 +9,10 @@ package db import ( "testing" "time" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) func TestVirtualMtimeRepo(t *testing.T) { - ldb, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } + ldb := OpenMemory() // A few repos so we can ensure they don't pollute each other repo1 := NewVirtualMtimeRepo(ldb, "folder1") diff --git a/lib/model/model.go b/lib/model/model.go index f305b709..06c6a5c1 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -33,7 +33,6 @@ import ( "github.com/syncthing/syncthing/lib/symlinks" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/versioner" - "github.com/syndtr/goleveldb/leveldb" "github.com/thejerf/suture" ) @@ -64,7 +63,7 @@ type Model struct { *suture.Supervisor cfg *config.Wrapper - db *leveldb.DB + db *db.Instance finder *db.BlockFinder progressEmitter *ProgressEmitter id protocol.DeviceID @@ -99,7 +98,7 @@ var ( // NewModel creates and starts a new model. The model starts in read-only mode, // where it sends index information to connected peers and responds to requests // for file data without altering the local folder in any way. -func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB, protectedFiles []string) *Model { +func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *db.Instance, protectedFiles []string) *Model { m := &Model{ Supervisor: suture.New("model", suture.Spec{ Log: func(line string) { diff --git a/lib/model/model_test.go b/lib/model/model_test.go index fa35adce..dcd28207 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -22,8 +22,6 @@ import ( "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) var device1, device2 protocol.DeviceID @@ -90,7 +88,7 @@ func init() { } func TestRequest(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) @@ -167,7 +165,7 @@ func BenchmarkIndex_100(b *testing.B) { } func benchmarkIndex(b *testing.B, nfiles int) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.StartFolderRO("default") @@ -196,7 +194,7 @@ func BenchmarkIndexUpdate_10000_1(b *testing.B) { } func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.StartFolderRO("default") @@ -261,7 +259,7 @@ func (FakeConnection) Statistics() protocol.Statistics { } func BenchmarkRequest(b *testing.B) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.ServeBackground() @@ -317,7 +315,7 @@ func TestDeviceRename(t *testing.T) { } cfg := config.Wrap("tmpconfig.xml", rawCfg) - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) fc := FakeConnection{ @@ -391,7 +389,7 @@ func TestClusterConfig(t *testing.T) { }, } - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(config.Wrap("/tmp/test", cfg), protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(cfg.Folders[0]) @@ -463,7 +461,7 @@ func TestIgnores(t *testing.T) { ioutil.WriteFile("testdata/.stfolder", nil, 0644) ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644) - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.StartFolderRO("default") @@ -538,7 +536,7 @@ func TestIgnores(t *testing.T) { } func TestRefuseUnknownBits(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.ServeBackground() @@ -576,7 +574,7 @@ func TestRefuseUnknownBits(t *testing.T) { } func TestROScanRecovery(t *testing.T) { - ldb, _ := leveldb.Open(storage.NewMemStorage(), nil) + ldb := db.OpenMemory() set := db.NewFileSet("default", ldb) set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ {Name: "dummyfile"}, @@ -660,7 +658,7 @@ func TestROScanRecovery(t *testing.T) { } func TestRWScanRecovery(t *testing.T) { - ldb, _ := leveldb.Open(storage.NewMemStorage(), nil) + ldb := db.OpenMemory() set := db.NewFileSet("default", ldb) set.Update(protocol.LocalDeviceID, []protocol.FileInfo{ {Name: "dummyfile"}, @@ -744,7 +742,7 @@ func TestRWScanRecovery(t *testing.T) { } func TestGlobalDirectoryTree(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.ServeBackground() @@ -994,7 +992,7 @@ func TestGlobalDirectoryTree(t *testing.T) { } func TestGlobalDirectorySelfFixing(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.ServeBackground() @@ -1168,7 +1166,7 @@ func BenchmarkTree_100_10(b *testing.B) { } func benchmarkTree(b *testing.B, n1, n2 int) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) m.ServeBackground() @@ -1186,7 +1184,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) { } func TestIgnoreDelete(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) // This folder should ignore external deletes diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index a09c674f..6eefac58 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -12,12 +12,10 @@ import ( "testing" "time" + "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/sync" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" ) func init() { @@ -69,7 +67,7 @@ func TestHandleFile(t *testing.T) { requiredFile := existingFile requiredFile.Blocks = blocks[1:] - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) // Update index @@ -125,7 +123,7 @@ func TestHandleFileWithTemp(t *testing.T) { requiredFile := existingFile requiredFile.Blocks = blocks[1:] - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) // Update index @@ -187,7 +185,7 @@ func TestCopierFinder(t *testing.T) { requiredFile.Blocks = blocks[1:] requiredFile.Name = "file2" - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) // Update index @@ -264,7 +262,7 @@ func TestCopierCleanup(t *testing.T) { return true } - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) @@ -313,7 +311,7 @@ func TestCopierCleanup(t *testing.T) { // Make sure that the copier routine hashes the content when asked, and pulls // if it fails to find the block. func TestLastResortPulling(t *testing.T) { - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) @@ -387,7 +385,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) { } defer os.Remove("testdata/" + defTempNamer.TempName("filex")) - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) @@ -480,7 +478,7 @@ func TestDeregisterOnFailInPull(t *testing.T) { } defer os.Remove("testdata/" + defTempNamer.TempName("filex")) - db, _ := leveldb.Open(storage.NewMemStorage(), nil) + db := db.OpenMemory() m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) m.AddFolder(defaultFolderConfig) diff --git a/lib/stats/device.go b/lib/stats/device.go index 9421b8f6..421d83e7 100644 --- a/lib/stats/device.go +++ b/lib/stats/device.go @@ -10,7 +10,6 @@ import ( "time" "github.com/syncthing/syncthing/lib/db" - "github.com/syndtr/goleveldb/leveldb" ) type DeviceStatistics struct { @@ -22,7 +21,7 @@ type DeviceStatisticsReference struct { device string } -func NewDeviceStatisticsReference(ldb *leveldb.DB, device string) *DeviceStatisticsReference { +func NewDeviceStatisticsReference(ldb *db.Instance, device string) *DeviceStatisticsReference { prefix := string(db.KeyTypeDeviceStatistic) + device return &DeviceStatisticsReference{ ns: db.NewNamespacedKV(ldb, prefix), diff --git a/lib/stats/folder.go b/lib/stats/folder.go index 96db7174..c1f36182 100644 --- a/lib/stats/folder.go +++ b/lib/stats/folder.go @@ -10,7 +10,6 @@ import ( "time" "github.com/syncthing/syncthing/lib/db" - "github.com/syndtr/goleveldb/leveldb" ) type FolderStatistics struct { @@ -28,7 +27,7 @@ type LastFile struct { Deleted bool `json:"deleted"` } -func NewFolderStatisticsReference(ldb *leveldb.DB, folder string) *FolderStatisticsReference { +func NewFolderStatisticsReference(ldb *db.Instance, folder string) *FolderStatisticsReference { prefix := string(db.KeyTypeFolderStatistic) + folder return &FolderStatisticsReference{ ns: db.NewNamespacedKV(ldb, prefix),