Implement database abstraction, error checking (ref #5907) (#6107)

This PR does two things, because one lead to the other:

- Move the leveldb specific stuff into a small "backend" package that
defines a backend interface and the leveldb implementation. This allows,
potentially, in the future, switching the db implementation so another
KV store should we wish to do so.

- Add proper error handling all along the way. The db and backend
packages are now errcheck clean. However, I drew the line at modifying
the FileSet API in order to keep this manageable and not continue
refactoring all of the rest of Syncthing. As such, the FileSet methods
still panic on database errors, except for the "database is closed"
error which is instead handled by silently returning as quickly as
possible, with the assumption that we're anyway "on the way out".

lib/db/backend/backend.go

@@ -0,0 +1,170 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"sync"
+)
+
+// The Reader interface specifies the read-only operations available on the
+// main database and on read-only transactions (snapshots). Note that when
+// called directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Reader interface {
+	Get(key []byte) ([]byte, error)
+	NewPrefixIterator(prefix []byte) (Iterator, error)
+	NewRangeIterator(first, last []byte) (Iterator, error)
+}
+
+// The Writer interface specifies the mutating operations available on the
+// main database and on writable transactions. Note that when called
+// directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Writer interface {
+	Put(key, val []byte) error
+	Delete(key []byte) error
+}
+
+// The ReadTransaction interface specifies the operations on read-only
+// transactions. Every ReadTransaction must be released when no longer
+// required.
+type ReadTransaction interface {
+	Reader
+	Release()
+}
+
+// The WriteTransaction interface specifies the operations on writable
+// transactions. Every WriteTransaction must be either committed or released
+// (i.e., discarded) when no longer required. No further operations must be
+// performed after release or commit (regardless of whether commit succeeded),
+// with one exception -- it's fine to release an already committed or released
+// transaction.
+//
+// A Checkpoint is a potential partial commit of the transaction so far, for
+// purposes of saving memory when transactions are in-RAM. Note that
+// transactions may be checkpointed *anyway* even if this is not called, due to
+// resource constraints, but this gives you a chance to decide when.
+type WriteTransaction interface {
+	ReadTransaction
+	Writer
+	Checkpoint() error
+	Commit() error
+}
+
+// The Iterator interface specifies the operations available on iterators
+// returned by NewPrefixIterator and NewRangeIterator. The iterator pattern
+// is to loop while Next returns true, then check Error after the loop. Next
+// will return false when iteration is complete (Error() == nil) or when
+// there is an error preventing iteration, which is then returned by
+// Error(). For example:
+//
+//     it, err := db.NewPrefixIterator(nil)
+//     if err != nil {
+//         // problem preventing iteration
+//     }
+//     defer it.Release()
+//     for it.Next() {
+//         // ...
+//     }
+//     if err := it.Error(); err != nil {
+//         // there was a database problem while iterating
+//     }
+//
+// An iterator must be Released when no longer required. The Error method
+// can be called either before or after Release with the same results. If an
+// iterator was created in a transaction (whether read-only or write) it
+// must be released before the transaction is released (or committed).
+type Iterator interface {
+	Next() bool
+	Key() []byte
+	Value() []byte
+	Error() error
+	Release()
+}
+
+// The Backend interface represents the main database handle. It supports
+// both read/write operations and opening read-only or writable
+// transactions. Depending on the actual implementation, individual
+// read/write operations may be implicitly wrapped in transactions, making
+// them perform quite badly when used repeatedly. For bulk operations,
+// consider always using a transaction of the appropriate type. The
+// transaction isolation level is "read committed" - there are no dirty
+// reads.
+type Backend interface {
+	Reader
+	Writer
+	NewReadTransaction() (ReadTransaction, error)
+	NewWriteTransaction() (WriteTransaction, error)
+	Close() error
+}
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/config.Tuning!
+	TuningAuto Tuning = iota
+	TuningSmall
+	TuningLarge
+)
+
+func Open(path string, tuning Tuning) (Backend, error) {
+	return OpenLevelDB(path, tuning)
+}
+
+func OpenMemory() Backend {
+	return OpenLevelDBMemory()
+}
+
+type errClosed struct{}
+
+func (errClosed) Error() string { return "database is closed" }
+
+type errNotFound struct{}
+
+func (errNotFound) Error() string { return "key not found" }
+
+func IsClosed(err error) bool {
+	if _, ok := err.(errClosed); ok {
+		return true
+	}
+	if _, ok := err.(*errClosed); ok {
+		return true
+	}
+	return false
+}
+
+func IsNotFound(err error) bool {
+	if _, ok := err.(errNotFound); ok {
+		return true
+	}
+	if _, ok := err.(*errNotFound); ok {
+		return true
+	}
+	return false
+}
+
+// releaser manages counting on top of a waitgroup
+type releaser struct {
+	wg   *sync.WaitGroup
+	once *sync.Once
+}
+
+func newReleaser(wg *sync.WaitGroup) *releaser {
+	wg.Add(1)
+	return &releaser{
+		wg:   wg,
+		once: new(sync.Once),
+	}
+}
+
+func (r releaser) Release() {
+	// We use the Once because we may get called multiple times from
+	// Commit() and deferred Release().
+	r.once.Do(func() {
+		r.wg.Done()
+	})
+}

lib/db/backend/backend_test.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import "testing"
+
+// testBackendBehavior is the generic test suite that must be fulfilled by
+// every backend implementation. It should be called by each implementation
+// as (part of) their test suite.
+func testBackendBehavior(t *testing.T, open func() Backend) {
+	t.Run("WriteIsolation", func(t *testing.T) { testWriteIsolation(t, open) })
+	t.Run("DeleteNonexisten", func(t *testing.T) { testDeleteNonexistent(t, open) })
+}
+
+func testWriteIsolation(t *testing.T, open func() Backend) {
+	// Values written during a transaction should not be read back, our
+	// updateGlobal depends on this.
+
+	db := open()
+	defer db.Close()
+
+	// Sanity check
+	_ = db.Put([]byte("a"), []byte("a"))
+	v, _ := db.Get([]byte("a"))
+	if string(v) != "a" {
+		t.Fatal("read back should work")
+	}
+
+	// Now in a transaction we should still see the old value
+	tx, _ := db.NewWriteTransaction()
+	defer tx.Release()
+	_ = tx.Put([]byte("a"), []byte("b"))
+	v, _ = tx.Get([]byte("a"))
+	if string(v) != "a" {
+		t.Fatal("read in transaction should read the old value")
+	}
+}
+
+func testDeleteNonexistent(t *testing.T, open func() Backend) {
+	// Deleting a non-existent key is not an error
+
+	db := open()
+	defer db.Close()
+
+	err := db.Delete([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+}

lib/db/backend/debug.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syncthing/syncthing/lib/logger"
+)
+
+var (
+	l = logger.DefaultLogger.NewFacility("backend", "The database backend")
+)

lib/db/backend/leveldb_backend.go

@@ -0,0 +1,173 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const (
+	// Never flush transactions smaller than this, even on Checkpoint()
+	dbFlushBatchMin = 1 << MiB
+	// Once a transaction reaches this size, flush it unconditionally.
+	dbFlushBatchMax = 128 << MiB
+)
+
+// leveldbBackend implements Backend on top of a leveldb
+type leveldbBackend struct {
+	ldb     *leveldb.DB
+	closeWG sync.WaitGroup
+}
+
+func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
+	return b.newSnapshot()
+}
+
+func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
+	snap, err := b.ldb.GetSnapshot()
+	if err != nil {
+		return leveldbSnapshot{}, wrapLeveldbErr(err)
+	}
+	return leveldbSnapshot{
+		snap: snap,
+		rel:  newReleaser(&b.closeWG),
+	}, nil
+}
+
+func (b *leveldbBackend) NewWriteTransaction() (WriteTransaction, error) {
+	snap, err := b.newSnapshot()
+	if err != nil {
+		return nil, err // already wrapped
+	}
+	return &leveldbTransaction{
+		leveldbSnapshot: snap,
+		ldb:             b.ldb,
+		batch:           new(leveldb.Batch),
+		rel:             newReleaser(&b.closeWG),
+	}, nil
+}
+
+func (b *leveldbBackend) Close() error {
+	b.closeWG.Wait()
+	return wrapLeveldbErr(b.ldb.Close())
+}
+
+func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
+	val, err := b.ldb.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return b.ldb.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (b *leveldbBackend) Put(key, val []byte) error {
+	return wrapLeveldbErr(b.ldb.Put(key, val, nil))
+}
+
+func (b *leveldbBackend) Delete(key []byte) error {
+	return wrapLeveldbErr(b.ldb.Delete(key, nil))
+}
+
+// leveldbSnapshot implements backend.ReadTransaction
+type leveldbSnapshot struct {
+	snap *leveldb.Snapshot
+	rel  *releaser
+}
+
+func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
+	val, err := l.snap.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (l leveldbSnapshot) Release() {
+	l.snap.Release()
+	l.rel.Release()
+}
+
+// leveldbTransaction implements backend.WriteTransaction using a batch (not
+// an actual leveldb transaction)
+type leveldbTransaction struct {
+	leveldbSnapshot
+	ldb   *leveldb.DB
+	batch *leveldb.Batch
+	rel   *releaser
+}
+
+func (t *leveldbTransaction) Delete(key []byte) error {
+	t.batch.Delete(key)
+	return t.checkFlush(dbFlushBatchMax)
+}
+
+func (t *leveldbTransaction) Put(key, val []byte) error {
+	t.batch.Put(key, val)
+	return t.checkFlush(dbFlushBatchMax)
+}
+
+func (t *leveldbTransaction) Checkpoint() error {
+	return t.checkFlush(dbFlushBatchMin)
+}
+
+func (t *leveldbTransaction) Commit() error {
+	err := wrapLeveldbErr(t.flush())
+	t.leveldbSnapshot.Release()
+	t.rel.Release()
+	return err
+}
+
+func (t *leveldbTransaction) Release() {
+	t.leveldbSnapshot.Release()
+	t.rel.Release()
+}
+
+// checkFlush flushes and resets the batch if its size exceeds the given size.
+func (t *leveldbTransaction) checkFlush(size int) error {
+	if len(t.batch.Dump()) < size {
+		return nil
+	}
+	return t.flush()
+}
+
+func (t *leveldbTransaction) flush() error {
+	if t.batch.Len() == 0 {
+		return nil
+	}
+	if err := t.ldb.Write(t.batch, nil); err != nil {
+		return wrapLeveldbErr(err)
+	}
+	t.batch.Reset()
+	return nil
+}
+
+// wrapLeveldbErr wraps errors so that the backend package can recognize them
+func wrapLeveldbErr(err error) error {
+	if err == nil {
+		return nil
+	}
+	if err == leveldb.ErrClosed {
+		return errClosed{}
+	}
+	if err == leveldb.ErrNotFound {
+		return errNotFound{}
+	}
+	return err
+}

lib/db/backend/leveldb_open.go

@@ -0,0 +1,226 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const (
+	dbMaxOpenFiles = 100
+
+	// A large database is > 200 MiB. It's a mostly arbitrary value, but
+	// it's also the case that each file is 2 MiB by default and when we
+	// have dbMaxOpenFiles of them we will need to start thrashing fd:s.
+	// Switching to large database settings causes larger files to be used
+	// when compacting, reducing the number.
+	dbLargeThreshold = dbMaxOpenFiles * (2 << MiB)
+
+	KiB = 10
+	MiB = 20
+)
+
+// Open attempts to open the database at the given location, and runs
+// recovery on it if opening fails. Worst case, if recovery is not possible,
+// the database is erased and created from scratch.
+func OpenLevelDB(location string, tuning Tuning) (Backend, error) {
+	opts := optsFor(location, tuning)
+	ldb, err := open(location, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &leveldbBackend{ldb: ldb}, nil
+}
+
+// OpenRO attempts to open the database at the given location, read only.
+func OpenLevelDBRO(location string) (Backend, error) {
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: dbMaxOpenFiles,
+		ReadOnly:               true,
+	}
+	ldb, err := open(location, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &leveldbBackend{ldb: ldb}, nil
+}
+
+// OpenMemory returns a new Lowlevel referencing an in-memory database.
+func OpenLevelDBMemory() Backend {
+	ldb, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	return &leveldbBackend{ldb: ldb}
+}
+
+// optsFor returns the database options to use when opening a database with
+// the given location and tuning. Settings can be overridden by debug
+// environment variables.
+func optsFor(location string, tuning Tuning) *opt.Options {
+	large := false
+	switch tuning {
+	case TuningLarge:
+		large = true
+	case TuningAuto:
+		large = dbIsLarge(location)
+	}
+
+	var (
+		// Set defaults used for small databases.
+		defaultBlockCacheCapacity            = 0 // 0 means let leveldb use default
+		defaultBlockSize                     = 0
+		defaultCompactionTableSize           = 0
+		defaultCompactionTableSizeMultiplier = 0
+		defaultWriteBuffer                   = 16 << MiB                      // increased from leveldb default of 4 MiB
+		defaultCompactionL0Trigger           = opt.DefaultCompactionL0Trigger // explicit because we use it as base for other stuff
+	)
+
+	if large {
+		// Change the parameters for better throughput at the price of some
+		// RAM and larger files. This results in larger batches of writes
+		// and compaction at a lower frequency.
+		l.Infoln("Using large-database tuning")
+
+		defaultBlockCacheCapacity = 64 << MiB
+		defaultBlockSize = 64 << KiB
+		defaultCompactionTableSize = 16 << MiB
+		defaultCompactionTableSizeMultiplier = 20 // 2.0 after division by ten
+		defaultWriteBuffer = 64 << MiB
+		defaultCompactionL0Trigger = 8 // number of l0 files
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", defaultBlockCacheCapacity),
+		BlockCacheEvictRemoved:        debugEnvValue("BlockCacheEvictRemoved", 0) != 0,
+		BlockRestartInterval:          debugEnvValue("BlockRestartInterval", 0),
+		BlockSize:                     debugEnvValue("BlockSize", defaultBlockSize),
+		CompactionExpandLimitFactor:   debugEnvValue("CompactionExpandLimitFactor", 0),
+		CompactionGPOverlapsFactor:    debugEnvValue("CompactionGPOverlapsFactor", 0),
+		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger),
+		CompactionSourceLimitFactor:   debugEnvValue("CompactionSourceLimitFactor", 0),
+		CompactionTableSize:           debugEnvValue("CompactionTableSize", defaultCompactionTableSize),
+		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", defaultCompactionTableSizeMultiplier)) / 10.0,
+		CompactionTotalSize:           debugEnvValue("CompactionTotalSize", 0),
+		CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
+		DisableBufferPool:             debugEnvValue("DisableBufferPool", 0) != 0,
+		DisableBlockCache:             debugEnvValue("DisableBlockCache", 0) != 0,
+		DisableCompactionBackoff:      debugEnvValue("DisableCompactionBackoff", 0) != 0,
+		DisableLargeBatchTransaction:  debugEnvValue("DisableLargeBatchTransaction", 0) != 0,
+		NoSync:                        debugEnvValue("NoSync", 0) != 0,
+		NoWriteMerge:                  debugEnvValue("NoWriteMerge", 0) != 0,
+		OpenFilesCacheCapacity:        debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
+		WriteBuffer:                   debugEnvValue("WriteBuffer", defaultWriteBuffer),
+		// The write slowdown and pause can be overridden, but even if they
+		// are not and the compaction trigger is overridden we need to
+		// adjust so that we don't pause writes for L0 compaction before we
+		// even *start* L0 compaction...
+		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
+		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
+	}
+
+	return opts
+}
+
+func open(location string, opts *opt.Options) (*leveldb.DB, error) {
+	db, err := leveldb.OpenFile(location, opts)
+	if leveldbIsCorrupted(err) {
+		db, err = leveldb.RecoverFile(location, opts)
+	}
+	if leveldbIsCorrupted(err) {
+		// The database is corrupted, and we've tried to recover it but it
+		// didn't work. At this point there isn't much to do beyond dropping
+		// the database and reindexing...
+		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
+		if err := os.RemoveAll(location); err != nil {
+			return nil, errorSuggestion{err, "failed to delete corrupted database"}
+		}
+		db, err = leveldb.OpenFile(location, opts)
+	}
+	if err != nil {
+		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
+	}
+
+	if debugEnvValue("CompactEverything", 0) != 0 {
+		if err := db.CompactRange(util.Range{}); err != nil {
+			l.Warnln("Compacting database:", err)
+		}
+	}
+
+	return db, nil
+}
+
+func debugEnvValue(key string, def int) int {
+	v, err := strconv.ParseInt(os.Getenv("STDEBUG_"+key), 10, 63)
+	if err != nil {
+		return def
+	}
+	return int(v)
+}
+
+// A "better" version of leveldb's errors.IsCorrupted.
+func leveldbIsCorrupted(err error) bool {
+	switch {
+	case err == nil:
+		return false
+
+	case errors.IsCorrupted(err):
+		return true
+
+	case strings.Contains(err.Error(), "corrupted"):
+		return true
+	}
+
+	return false
+}
+
+// dbIsLarge returns whether the estimated size of the database at location
+// is large enough to warrant optimization for large databases.
+func dbIsLarge(location string) bool {
+	if ^uint(0)>>63 == 0 {
+		// We're compiled for a 32 bit architecture. We've seen trouble with
+		// large settings there.
+		// (https://forum.syncthing.net/t/many-small-ldb-files-with-database-tuning/13842)
+		return false
+	}
+
+	dir, err := os.Open(location)
+	if err != nil {
+		return false
+	}
+
+	fis, err := dir.Readdir(-1)
+	if err != nil {
+		return false
+	}
+
+	var size int64
+	for _, fi := range fis {
+		if fi.Name() == "LOG" {
+			// don't count the size
+			continue
+		}
+		size += fi.Size()
+	}
+
+	return size > dbLargeThreshold
+}
+
+type errorSuggestion struct {
+	inner      error
+	suggestion string
+}
+
+func (e errorSuggestion) Error() string {
+	return fmt.Sprintf("%s (%s)", e.inner.Error(), e.suggestion)
+}

lib/db/backend/leveldb_test.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import "testing"
+
+func TestLevelDBBackendBehavior(t *testing.T) {
+	testBackendBehavior(t, OpenLevelDBMemory)
+}