diff --git a/internal/db/blockmap.go b/internal/db/blockmap.go index cfc4a135..9ee4b8c9 100644 --- a/internal/db/blockmap.go +++ b/internal/db/blockmap.go @@ -199,7 +199,7 @@ func (f *BlockFinder) Fix(folder, file string, index int32, oldHash, newHash []b // file name (variable size) func toBlockKey(hash []byte, folder, file string) []byte { o := make([]byte, 1+64+32+len(file)) - o[0] = keyTypeBlock + o[0] = KeyTypeBlock copy(o[1:], []byte(folder)) copy(o[1+64:], []byte(hash)) copy(o[1+64+32:], []byte(file)) @@ -210,7 +210,7 @@ func fromBlockKey(data []byte) (string, string) { if len(data) < 1+64+32+1 { panic("Incorrect key length") } - if data[0] != keyTypeBlock { + if data[0] != KeyTypeBlock { panic("Incorrect key type") } diff --git a/internal/db/leveldb.go b/internal/db/leveldb.go index 34f2e465..1b02840f 100644 --- a/internal/db/leveldb.go +++ b/internal/db/leveldb.go @@ -50,9 +50,11 @@ func clock(v int64) int64 { } const ( - keyTypeDevice = iota - keyTypeGlobal - keyTypeBlock + KeyTypeDevice = iota + KeyTypeGlobal + KeyTypeBlock + KeyTypeDeviceStatistic + KeyTypeFolderStatistic ) type fileVersion struct { @@ -112,7 +114,7 @@ const batchFlushSize = 64 // name (variable size) func deviceKey(folder, device, file []byte) []byte { k := make([]byte, 1+64+32+len(file)) - k[0] = keyTypeDevice + k[0] = KeyTypeDevice if len(folder) > 64 { panic("folder name too long") } @@ -145,7 +147,7 @@ func deviceKeyDevice(key []byte) []byte { // name (variable size) func globalKey(folder, file []byte) []byte { k := make([]byte, 1+64+len(file)) - k[0] = keyTypeGlobal + k[0] = KeyTypeGlobal if len(folder) > 64 { panic("folder name too long") } @@ -915,7 +917,7 @@ func ldbListFolders(db *leveldb.DB) []string { snap.Release() }() - dbi := snap.NewIterator(util.BytesPrefix([]byte{keyTypeGlobal}), nil) + dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil) defer dbi.Release() folderExists := make(map[string]bool) @@ -953,7 +955,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) { }() // Remove all items related to the given folder from the device->file bucket - dbi := snap.NewIterator(util.BytesPrefix([]byte{keyTypeDevice}), nil) + dbi := snap.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) for dbi.Next() { itemFolder := deviceKeyFolder(dbi.Key()) if bytes.Compare(folder, itemFolder) == 0 { @@ -963,7 +965,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) { dbi.Release() // Remove all items related to the given folder from the global bucket - dbi = snap.NewIterator(util.BytesPrefix([]byte{keyTypeGlobal}), nil) + dbi = snap.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil) for dbi.Next() { itemFolder := globalKeyFolder(dbi.Key()) if bytes.Compare(folder, itemFolder) == 0 { diff --git a/internal/db/namespaced.go b/internal/db/namespaced.go new file mode 100644 index 00000000..14408161 --- /dev/null +++ b/internal/db/namespaced.go @@ -0,0 +1,106 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +package db + +import ( + "encoding/binary" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +// NamespacedKV is a simple key-value store using a specific namespace within +// a leveldb. +type NamespacedKV struct { + db *leveldb.DB + prefix []byte +} + +// NewNamespacedKV returns a new NamespacedKV that lives in the namespace +// specified by the prefix. +func NewNamespacedKV(db *leveldb.DB, prefix string) *NamespacedKV { + return &NamespacedKV{ + db: db, + prefix: []byte(prefix), + } +} + +// PutInt64 stores a new int64. Any existing value (even if of another type) +// is overwritten. +func (n *NamespacedKV) PutInt64(key string, val int64) { + keyBs := append(n.prefix, []byte(key)...) + var valBs [8]byte + binary.BigEndian.PutUint64(valBs[:], uint64(val)) + n.db.Put(keyBs, valBs[:], nil) +} + +// Int64 returns the stored value interpreted as an int64 and a boolean that +// is false if no value was stored at the key. +func (n *NamespacedKV) Int64(key string) (int64, bool) { + keyBs := append(n.prefix, []byte(key)...) + valBs, err := n.db.Get(keyBs, nil) + if err != nil { + return 0, false + } + val := binary.BigEndian.Uint64(valBs) + return int64(val), true +} + +// PutTime stores a new time.Time. Any existing value (even if of another +// type) is overwritten. +func (n *NamespacedKV) PutTime(key string, val time.Time) { + keyBs := append(n.prefix, []byte(key)...) + valBs, _ := val.MarshalBinary() // never returns an error + n.db.Put(keyBs, valBs, nil) +} + +// Time returns the stored value interpreted as a time.Time and a boolean +// that is false if no value was stored at the key. +func (n NamespacedKV) Time(key string) (time.Time, bool) { + var t time.Time + keyBs := append(n.prefix, []byte(key)...) + valBs, err := n.db.Get(keyBs, nil) + if err != nil { + return t, false + } + err = t.UnmarshalBinary(valBs) + return t, err == nil +} + +// PutString stores a new string. Any existing value (even if of another type) +// is overwritten. +func (n *NamespacedKV) PutString(key, val string) { + keyBs := append(n.prefix, []byte(key)...) + n.db.Put(keyBs, []byte(val), nil) +} + +// String returns the stored value interpreted as a string and a boolean that +// is false if no value was stored at the key. +func (n NamespacedKV) String(key string) (string, bool) { + keyBs := append(n.prefix, []byte(key)...) + valBs, err := n.db.Get(keyBs, nil) + if err != nil { + return "", false + } + return string(valBs), true +} + +// Delete deletes the specified key. It is allowed to delete a nonexistent +// key. +func (n NamespacedKV) Delete(key string) { + keyBs := append(n.prefix, []byte(key)...) + n.db.Delete(keyBs, nil) +} diff --git a/internal/db/namespaced_test.go b/internal/db/namespaced_test.go new file mode 100644 index 00000000..b42ce6f1 --- /dev/null +++ b/internal/db/namespaced_test.go @@ -0,0 +1,101 @@ +// Copyright (C) 2014 The Syncthing Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +package db + +import ( + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" +) + +func TestNamespacedInt(t *testing.T) { + ldb, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + t.Fatal(err) + } + + n1 := NewNamespacedKV(ldb, "foo") + n2 := NewNamespacedKV(ldb, "bar") + + // Key is missing to start with + + if v, ok := n1.Int64("test"); v != 0 || ok { + t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok) + } + + n1.PutInt64("test", 42) + + // It should now exist in n1 + + if v, ok := n1.Int64("test"); v != 42 || !ok { + t.Errorf("Incorrect return v %v != 42 || ok %v != true", v, ok) + } + + // ... but not in n2, which is in a different namespace + + if v, ok := n2.Int64("test"); v != 0 || ok { + t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok) + } + + n1.Delete("test") + + // It should no longer exist + + if v, ok := n1.Int64("test"); v != 0 || ok { + t.Errorf("Incorrect return v %v != 0 || ok %v != false", v, ok) + } +} + +func TestNamespacedTime(t *testing.T) { + ldb, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + t.Fatal(err) + } + + n1 := NewNamespacedKV(ldb, "foo") + + if v, ok := n1.Time("test"); v != (time.Time{}) || ok { + t.Errorf("Incorrect return v %v != %v || ok %v != false", v, time.Time{}, ok) + } + + now := time.Now() + n1.PutTime("test", now) + + if v, ok := n1.Time("test"); v != now || !ok { + t.Errorf("Incorrect return v %v != %v || ok %v != true", v, now, ok) + } +} + +func TestNamespacedString(t *testing.T) { + ldb, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + t.Fatal(err) + } + + n1 := NewNamespacedKV(ldb, "foo") + + if v, ok := n1.String("test"); v != "" || ok { + t.Errorf("Incorrect return v %q != \"\" || ok %v != false", v, ok) + } + + n1.PutString("test", "yo") + + if v, ok := n1.String("test"); v != "yo" || !ok { + t.Errorf("Incorrect return v %q != \"yo\" || ok %v != true", v, ok) + } +} diff --git a/internal/stats/device.go b/internal/stats/device.go index faccd9ef..0953db40 100644 --- a/internal/stats/device.go +++ b/internal/stats/device.go @@ -19,91 +19,45 @@ import ( "time" "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/internal/db" "github.com/syndtr/goleveldb/leveldb" ) -const ( - deviceStatisticTypeLastSeen = iota -) - -var deviceStatisticsTypes = []byte{ - deviceStatisticTypeLastSeen, -} - type DeviceStatistics struct { LastSeen time.Time } type DeviceStatisticsReference struct { - db *leveldb.DB + ns *db.NamespacedKV device protocol.DeviceID } -func NewDeviceStatisticsReference(db *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference { +func NewDeviceStatisticsReference(ldb *leveldb.DB, device protocol.DeviceID) *DeviceStatisticsReference { + prefix := string(db.KeyTypeDeviceStatistic) + device.String() return &DeviceStatisticsReference{ - db: db, + ns: db.NewNamespacedKV(ldb, prefix), device: device, } } -func (s *DeviceStatisticsReference) key(stat byte) []byte { - k := make([]byte, 1+1+32) - k[0] = keyTypeDeviceStatistic - k[1] = stat - copy(k[1+1:], s.device[:]) - return k -} - func (s *DeviceStatisticsReference) GetLastSeen() time.Time { - value, err := s.db.Get(s.key(deviceStatisticTypeLastSeen), nil) - if err != nil { - if err != leveldb.ErrNotFound { - l.Warnln("DeviceStatisticsReference: Failed loading last seen value for", s.device, ":", err) - } - return time.Unix(0, 0) - } - - rtime := time.Time{} - err = rtime.UnmarshalBinary(value) - if err != nil { - l.Warnln("DeviceStatisticsReference: Failed parsing last seen value for", s.device, ":", err) + t, ok := s.ns.Time("lastSeen") + if !ok { + // The default here is 1970-01-01 as opposed to the default + // time.Time{} from s.ns return time.Unix(0, 0) } if debug { - l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, rtime) + l.Debugln("stats.DeviceStatisticsReference.GetLastSeen:", s.device, t) } - return rtime + return t } func (s *DeviceStatisticsReference) WasSeen() { if debug { l.Debugln("stats.DeviceStatisticsReference.WasSeen:", s.device) } - value, err := time.Now().MarshalBinary() - if err != nil { - l.Warnln("DeviceStatisticsReference: Failed serializing last seen value for", s.device, ":", err) - return - } - - err = s.db.Put(s.key(deviceStatisticTypeLastSeen), value, nil) - if err != nil { - l.Warnln("Failed serializing last seen value for", s.device, ":", err) - } -} - -// Never called, maybe because it's worth while to keep the data -// or maybe because we have no easy way of knowing that a device has been removed. -func (s *DeviceStatisticsReference) Delete() error { - for _, stype := range deviceStatisticsTypes { - err := s.db.Delete(s.key(stype), nil) - if debug && err == nil { - l.Debugln("stats.DeviceStatisticsReference.Delete:", s.device, stype) - } - if err != nil && err != leveldb.ErrNotFound { - return err - } - } - return nil + s.ns.PutTime("lastSeen", time.Now()) } func (s *DeviceStatisticsReference) GetStatistics() DeviceStatistics { diff --git a/internal/stats/folder.go b/internal/stats/folder.go index 0a4c199e..b253f912 100644 --- a/internal/stats/folder.go +++ b/internal/stats/folder.go @@ -16,118 +16,59 @@ package stats import ( - "encoding/binary" "time" + "github.com/syncthing/syncthing/internal/db" "github.com/syndtr/goleveldb/leveldb" ) -const ( - folderStatisticTypeLastFile = iota -) - -var folderStatisticsTypes = []byte{ - folderStatisticTypeLastFile, -} - type FolderStatistics struct { - LastFile *LastFile + LastFile LastFile } type FolderStatisticsReference struct { - db *leveldb.DB + ns *db.NamespacedKV folder string } -func NewFolderStatisticsReference(db *leveldb.DB, folder string) *FolderStatisticsReference { - return &FolderStatisticsReference{ - db: db, - folder: folder, - } -} - -func (s *FolderStatisticsReference) key(stat byte) []byte { - k := make([]byte, 1+1+64) - k[0] = keyTypeFolderStatistic - k[1] = stat - copy(k[1+1:], s.folder[:]) - return k -} - -func (s *FolderStatisticsReference) GetLastFile() *LastFile { - value, err := s.db.Get(s.key(folderStatisticTypeLastFile), nil) - if err != nil { - if err != leveldb.ErrNotFound { - l.Warnln("FolderStatisticsReference: Failed loading last file filename value for", s.folder, ":", err) - } - return nil - } - - file := LastFile{} - err = file.UnmarshalBinary(value) - if err != nil { - l.Warnln("FolderStatisticsReference: Failed loading last file value for", s.folder, ":", err) - return nil - } - return &file -} - -func (s *FolderStatisticsReference) ReceivedFile(filename string) { - f := LastFile{ - Filename: filename, - At: time.Now(), - } - if debug { - l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder) - } - - value, err := f.MarshalBinary() - if err != nil { - l.Warnln("FolderStatisticsReference: Failed serializing last file value for", s.folder, ":", err) - return - } - - err = s.db.Put(s.key(folderStatisticTypeLastFile), value, nil) - if err != nil { - l.Warnln("Failed update last file value for", s.folder, ":", err) - } -} - -// Never called, maybe because it's worth while to keep the data -// or maybe because we have no easy way of knowing that a folder has been removed. -func (s *FolderStatisticsReference) Delete() error { - for _, stype := range folderStatisticsTypes { - err := s.db.Delete(s.key(stype), nil) - if debug && err == nil { - l.Debugln("stats.FolderStatisticsReference.Delete:", s.folder, stype) - } - if err != nil && err != leveldb.ErrNotFound { - return err - } - } - return nil -} - -func (s *FolderStatisticsReference) GetStatistics() FolderStatistics { - return FolderStatistics{ - LastFile: s.GetLastFile(), - } -} - type LastFile struct { At time.Time Filename string } -func (f *LastFile) MarshalBinary() ([]byte, error) { - buf := make([]byte, 8+len(f.Filename)) - binary.BigEndian.PutUint64(buf[:8], uint64(f.At.Unix())) - copy(buf[8:], []byte(f.Filename)) - return buf, nil +func NewFolderStatisticsReference(ldb *leveldb.DB, folder string) *FolderStatisticsReference { + prefix := string(db.KeyTypeFolderStatistic) + folder + return &FolderStatisticsReference{ + ns: db.NewNamespacedKV(ldb, prefix), + folder: folder, + } } -func (f *LastFile) UnmarshalBinary(buf []byte) error { - f.At = time.Unix(int64(binary.BigEndian.Uint64(buf[:8])), 0) - f.Filename = string(buf[8:]) - return nil +func (s *FolderStatisticsReference) GetLastFile() LastFile { + at, ok := s.ns.Time("lastFileAt") + if !ok { + return LastFile{} + } + file, ok := s.ns.String("lastFileName") + if !ok { + return LastFile{} + } + return LastFile{ + At: at, + Filename: file, + } +} + +func (s *FolderStatisticsReference) ReceivedFile(filename string) { + if debug { + l.Debugln("stats.FolderStatisticsReference.ReceivedFile:", s.folder, filename) + } + s.ns.PutTime("lastFileAt", time.Now()) + s.ns.PutString("lastFileName", filename) +} + +func (s *FolderStatisticsReference) GetStatistics() FolderStatistics { + return FolderStatistics{ + LastFile: s.GetLastFile(), + } } diff --git a/internal/stats/leveldb.go b/internal/stats/leveldb.go deleted file mode 100644 index 77353bfe..00000000 --- a/internal/stats/leveldb.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . - -package stats - -// Same key space as files/leveldb.go keyType* constants -const ( - keyTypeDeviceStatistic = iota + 30 - keyTypeFolderStatistic -)