diff --git a/internal/files/blockmap.go b/internal/files/blockmap.go new file mode 100644 index 00000000..c24f1f03 --- /dev/null +++ b/internal/files/blockmap.go @@ -0,0 +1,201 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +// Package files provides a set type to track local/remote files with newness +// checks. We must do a certain amount of normalization in here. We will get +// fed paths with either native or wire-format separators and encodings +// depending on who calls us. We transform paths to wire-format (NFC and +// slashes) on the way to the database, and transform to native format +// (varying separator and encoding) on the way back out. + +package files + +import ( + "bytes" + "encoding/binary" + "sort" + "sync" + + "github.com/syncthing/syncthing/internal/config" + "github.com/syncthing/syncthing/internal/protocol" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" +) + +var blockFinder *BlockFinder + +type BlockMap struct { + db *leveldb.DB + folder string +} + +func NewBlockMap(db *leveldb.DB, folder string) *BlockMap { + return &BlockMap{ + db: db, + folder: folder, + } +} + +// Add files to the block map, ignoring any deleted or invalid files. +func (m *BlockMap) Add(files []protocol.FileInfo) error { + batch := new(leveldb.Batch) + buf := make([]byte, 4) + for _, file := range files { + if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() { + continue + } + + for i, block := range file.Blocks { + binary.BigEndian.PutUint32(buf, uint32(i)) + batch.Put(m.blockKey(block.Hash, file.Name), buf) + } + } + return m.db.Write(batch, nil) +} + +// Update block map state, removing any deleted or invalid files. +func (m *BlockMap) Update(files []protocol.FileInfo) error { + batch := new(leveldb.Batch) + buf := make([]byte, 4) + for _, file := range files { + if file.IsDirectory() { + continue + } + + if file.IsDeleted() || file.IsInvalid() { + for _, block := range file.Blocks { + batch.Delete(m.blockKey(block.Hash, file.Name)) + } + continue + } + + for i, block := range file.Blocks { + binary.BigEndian.PutUint32(buf, uint32(i)) + batch.Put(m.blockKey(block.Hash, file.Name), buf) + } + } + return m.db.Write(batch, nil) +} + +// Drop block map, removing all entries related to this block map from the db. +func (m *BlockMap) Drop() error { + batch := new(leveldb.Batch) + iter := m.db.NewIterator(util.BytesPrefix(m.blockKey(nil, "")[:1+64]), nil) + defer iter.Release() + for iter.Next() { + batch.Delete(iter.Key()) + } + if iter.Error() != nil { + return iter.Error() + } + return m.db.Write(batch, nil) +} + +func (m *BlockMap) blockKey(hash []byte, file string) []byte { + return toBlockKey(hash, m.folder, file) +} + +type BlockFinder struct { + db *leveldb.DB + folders []string + mut sync.RWMutex +} + +func NewBlockFinder(db *leveldb.DB, cfg *config.ConfigWrapper) *BlockFinder { + if blockFinder != nil { + return blockFinder + } + + f := &BlockFinder{ + db: db, + } + f.Changed(cfg.Raw()) + cfg.Subscribe(f) + return f +} + +// Implements config.Handler interface +func (f *BlockFinder) Changed(cfg config.Configuration) error { + folders := make([]string, len(cfg.Folders)) + for i, folder := range cfg.Folders { + folders[i] = folder.ID + } + + sort.Strings(folders) + + f.mut.Lock() + f.folders = folders + f.mut.Unlock() + + return nil +} + +// An iterator function which iterates over all matching blocks for the given +// hash. The iterator function has to return either true (if they are happy with +// the block) or false to continue iterating for whatever reason. +// The iterator finally returns the result, whether or not a satisfying block +// was eventually found. +func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) bool) bool { + f.mut.RLock() + folders := f.folders + f.mut.RUnlock() + for _, folder := range folders { + key := toBlockKey(hash, folder, "") + iter := f.db.NewIterator(util.BytesPrefix(key), nil) + defer iter.Release() + + for iter.Next() && iter.Error() == nil { + folder, file := fromBlockKey(iter.Key()) + index := binary.BigEndian.Uint32(iter.Value()) + if iterFn(folder, nativeFilename(file), index) { + return true + } + } + } + return false +} + +// m.blockKey returns a byte slice encoding the following information: +// keyTypeBlock (1 byte) +// folder (64 bytes) +// block hash (64 bytes) +// file name (variable size) +func toBlockKey(hash []byte, folder, file string) []byte { + o := make([]byte, 1+64+64+len(file)) + o[0] = keyTypeBlock + copy(o[1:], []byte(folder)) + copy(o[1+64:], []byte(hash)) + copy(o[1+64+64:], []byte(file)) + return o +} + +func fromBlockKey(data []byte) (string, string) { + if len(data) < 1+64+64+1 { + panic("Incorrect key length") + } + if data[0] != keyTypeBlock { + panic("Incorrect key type") + } + + file := string(data[1+64+64:]) + + slice := data[1 : 1+64] + izero := bytes.IndexByte(slice, 0) + if izero > -1 { + return string(slice[:izero]), file + } + return string(slice), file +} diff --git a/internal/files/blockmap_test.go b/internal/files/blockmap_test.go new file mode 100644 index 00000000..ea195261 --- /dev/null +++ b/internal/files/blockmap_test.go @@ -0,0 +1,235 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . + +package files + +import ( + "testing" + + "github.com/syncthing/syncthing/internal/config" + "github.com/syncthing/syncthing/internal/protocol" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" +) + +func genBlocks(n int) []protocol.BlockInfo { + b := make([]protocol.BlockInfo, n) + for i := range b { + h := make([]byte, 32) + for j := range h { + h[j] = byte(i + j) + } + b[i].Size = uint32(i) + b[i].Hash = h + } + return b +} + +var f1, f2, f3 protocol.FileInfo + +func init() { + blocks := genBlocks(30) + + f1 = protocol.FileInfo{ + Name: "f1", + Blocks: blocks[:10], + } + + f2 = protocol.FileInfo{ + Name: "f2", + Blocks: blocks[10:20], + } + + f3 = protocol.FileInfo{ + Name: "f3", + Blocks: blocks[20:], + } +} + +func setup() (*leveldb.DB, *BlockFinder) { + // Setup + + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + panic(err) + } + + wrapper := config.Wrap("", config.Configuration{}) + wrapper.SetFolder(config.FolderConfiguration{ + ID: "folder1", + }) + wrapper.SetFolder(config.FolderConfiguration{ + ID: "folder2", + }) + + return db, NewBlockFinder(db, wrapper) +} + +func dbEmpty(db *leveldb.DB) bool { + iter := db.NewIterator(nil, nil) + defer iter.Release() + if iter.Next() { + return false + } + return true +} + +func TestBlockMapAddUpdateWipe(t *testing.T) { + db, f := setup() + + if !dbEmpty(db) { + t.Fatal("db not empty") + } + + m := NewBlockMap(db, "folder1") + + f3.Flags |= protocol.FlagDirectory + + err := m.Add([]protocol.FileInfo{f1, f2, f3}) + if err != nil { + t.Fatal(err) + } + + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + if folder != "folder1" || file != "f1" || index != 0 { + t.Fatal("Mismatch") + } + return true + }) + + f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool { + if folder != "folder1" || file != "f2" || index != 0 { + t.Fatal("Mismatch") + } + return true + }) + + f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool { + t.Fatal("Unexpected block") + return true + }) + + f3.Flags = f1.Flags + f1.Flags |= protocol.FlagDeleted + f2.Flags |= protocol.FlagInvalid + + // Should remove + err = m.Update([]protocol.FileInfo{f1, f2, f3}) + if err != nil { + t.Fatal(err) + } + + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + t.Fatal("Unexpected block") + return false + }) + + f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool { + t.Fatal("Unexpected block") + return false + }) + + f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool { + if folder != "folder1" || file != "f3" || index != 0 { + t.Fatal("Mismatch") + } + return true + }) + + err = m.Drop() + if err != nil { + t.Fatal(err) + } + + if !dbEmpty(db) { + t.Fatal("db not empty") + } + + // Should not add + err = m.Add([]protocol.FileInfo{f1, f2}) + if err != nil { + t.Fatal(err) + } + + if !dbEmpty(db) { + t.Fatal("db not empty") + } + + f1.Flags = 0 + f2.Flags = 0 + f3.Flags = 0 +} + +func TestBlockMapFinderLookup(t *testing.T) { + db, f := setup() + + m1 := NewBlockMap(db, "folder1") + m2 := NewBlockMap(db, "folder2") + + err := m1.Add([]protocol.FileInfo{f1}) + if err != nil { + t.Fatal(err) + } + err = m2.Add([]protocol.FileInfo{f1}) + if err != nil { + t.Fatal(err) + } + + counter := 0 + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + counter++ + switch counter { + case 1: + if folder != "folder1" || file != "f1" || index != 0 { + t.Fatal("Mismatch") + } + case 2: + if folder != "folder2" || file != "f1" || index != 0 { + t.Fatal("Mismatch") + } + default: + t.Fatal("Unexpected block") + } + return false + }) + if counter != 2 { + t.Fatal("Incorrect count", counter) + } + + f1.Flags |= protocol.FlagDeleted + + err = m1.Update([]protocol.FileInfo{f1}) + if err != nil { + t.Fatal(err) + } + + counter = 0 + f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool { + counter++ + switch counter { + case 1: + if folder != "folder2" || file != "f1" || index != 0 { + t.Fatal("Mismatch") + } + default: + t.Fatal("Unexpected block") + } + return false + }) + if counter != 1 { + t.Fatal("Incorrect count") + } +} diff --git a/internal/files/leveldb.go b/internal/files/leveldb.go index 0ac1b833..06c8bba6 100644 --- a/internal/files/leveldb.go +++ b/internal/files/leveldb.go @@ -49,6 +49,7 @@ func clock(v uint64) uint64 { const ( keyTypeDevice = iota keyTypeGlobal + keyTypeBlock ) type fileVersion struct { diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 800787b9..b4714505 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -54,6 +54,10 @@ func (f FileInfo) IsInvalid() bool { return IsInvalid(f.Flags) } +func (f FileInfo) IsDirectory() bool { + return IsDirectory(f.Flags) +} + // Used for unmarshalling a FileInfo structure but skipping the actual block list type FileInfoTruncated struct { Name string // max:8192