perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/model: Verify request content against weak (and possibly strong) hash (#4767)

Browse Source
This commit is contained in:
Audrius Butkevicius
2018-05-05 09:24:44 +01:00
committed by Jakob Borg
parent 678c80ffe4
commit ef0dcea6a4
27 changed files with 518 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
lib/model/model.go
View File

@@ -7,6 +7,7 @@
package model
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
@@ -34,7 +35,6 @@ import (
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
"github.com/thejerf/suture"
)
@@ -1304,7 +1304,7 @@ func (m *Model) closeLocked(device protocol.DeviceID) {
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32, fromTemporary bool, buf []byte) error {
if offset < 0 {
return protocol.ErrInvalid
}
@@ -1362,8 +1362,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
// other than a regular file.
return protocol.ErrNoSuchFile
}
if err := readOffsetIntoBuf(folderFs, tempFn, offset, buf); err == nil {
err := readOffsetIntoBuf(folderFs, tempFn, offset, buf)
if err == nil && scanner.Validate(buf, hash, weakHash) {
return nil
}
// Fall through to reading from a non-temp file, just incase the temp
@@ -1382,9 +1382,55 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
return protocol.ErrGeneric
}
if !scanner.Validate(buf, hash, weakHash) {
m.recheckFile(deviceID, folderFs, folder, name, int(offset)/len(buf), hash)
return protocol.ErrNoSuchFile
}
return nil
}
func (m *Model) recheckFile(deviceID protocol.DeviceID, folderFs fs.Filesystem, folder, name string, blockIndex int, hash []byte) {
cf, ok := m.CurrentFolderFile(folder, name)
if !ok {
l.Debugf("%v recheckFile: %s: %q / %q: no current file", m, deviceID, folder, name)
return
}
if cf.IsDeleted() || cf.IsInvalid() || cf.IsSymlink() || cf.IsDirectory() {
l.Debugf("%v recheckFile: %s: %q / %q: not a regular file", m, deviceID, folder, name)
return
}
if blockIndex > len(cf.Blocks) {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: block index too far", m, deviceID, folder, name, blockIndex)
return
}
block := cf.Blocks[blockIndex]
// Seems to want a different version of the file, whatever.
if !bytes.Equal(block.Hash, hash) {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
return
}
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
// Something is fishy, invalidate the file and rescan it.
cf.Invalidate(m.shortID)
// Update the index and tell others
// The file will temporarily become invalid, which is ok as the content is messed up.
m.updateLocalsFromScanning(folder, []protocol.FileInfo{cf})
if err := m.ScanFolderSubdirs(folder, []string{name}); err != nil {
l.Debugf("%v recheckFile: %s: %q / %q rescan: %s", m, deviceID, folder, name, err)
} else {
l.Debugf("%v recheckFile: %s: %q / %q", m, deviceID, folder, name)
}
}
func (m *Model) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
@@ -1836,7 +1882,7 @@ func (m *Model) diskChangeDetected(folderCfg config.FolderConfiguration, files [
}
}
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
m.pmut.RLock()
nc, ok := m.conn[deviceID]
m.pmut.RUnlock()
@@ -1845,9 +1891,9 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
}
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t", m, deviceID, folder, name, offset, size, hash, fromTemporary)
l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x wh=%x ft=%t", m, deviceID, folder, name, offset, size, hash, weakHash, fromTemporary)
return nc.Request(folder, name, offset, size, hash, fromTemporary)
return nc.Request(folder, name, offset, size, hash, weakHash, fromTemporary)
}
func (m *Model) ScanFolders() map[string]error {
@@ -1973,7 +2019,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
Hashers: m.numHashers(folder),
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseWeakHashes: weakhash.Enabled,
UseLargeBlocks: folderCfg.UseLargeBlocks,
})

18
lib/model/model_test.go
View File

@@ -181,7 +181,7 @@ func TestRequest(t *testing.T) {
// Existing, shared file
bs = bs[:6]
err := m.Request(device1, "default", "foo", 0, nil, false, bs)
err := m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
if err != nil {
t.Error(err)
}
@@ -190,32 +190,32 @@ func TestRequest(t *testing.T) {
}
// Existing, nonshared file
err = m.Request(device2, "default", "foo", 0, nil, false, bs)
err = m.Request(device2, "default", "foo", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Nonexistent file
err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
err = m.Request(device1, "default", "nonexistent", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Shared folder, but disallowed file name
err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
err = m.Request(device1, "default", "../walk.go", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Negative offset
err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
err = m.Request(device1, "default", "foo", -4, nil, 0, false, bs[:0])
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
// Larger block than available
bs = bs[:42]
err = m.Request(device1, "default", "foo", 0, nil, false, bs)
err = m.Request(device1, "default", "foo", 0, nil, 0, false, bs)
if err == nil {
t.Error("Unexpected nil error on insecure file read")
}
@@ -357,7 +357,7 @@ func (f *fakeConnection) IndexUpdate(folder string, fs []protocol.FileInfo) erro
return nil
}
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
func (f *fakeConnection) Request(folder, name string, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
f.mut.Lock()
defer f.mut.Unlock()
if f.requestFn != nil {
@@ -485,7 +485,7 @@ func BenchmarkRequestOut(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, false)
data, err := m.requestGlobal(device1, "default", files[i%n].Name, 0, 32, nil, 0, false)
if err != nil {
b.Error(err)
}
@@ -513,7 +513,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, false, buf); err != nil {
if err := m.Request(device1, "default", "request/for/a/file/in/a/couple/of/dirs/128k", 0, nil, 0, false, buf); err != nil {
b.Error(err)
}
}

69
lib/model/requests_test.go
View File

@@ -93,7 +93,7 @@ func TestSymlinkTraversalRead(t *testing.T) {
// Request a file by traversing the symlink
buf := make([]byte, 10)
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, false, buf)
err := m.Request(device1, "default", "symlink/requests_test.go", 0, nil, 0, false, buf)
if err == nil || !bytes.Equal(buf, make([]byte, 10)) {
t.Error("Managed to traverse symlink")
}
@@ -464,6 +464,73 @@ func TestIssue4841(t *testing.T) {
}
}
func TestRescanIfHaveInvalidContent(t *testing.T) {
m, fc, tmpDir := setupModelWithConnection()
defer m.Stop()
defer os.RemoveAll(tmpDir)
payload := []byte("hello")
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
t.Fatal(err)
}
received := make(chan protocol.FileInfo)
fc.mut.Lock()
fc.indexFn = func(folder string, fs []protocol.FileInfo) {
if len(fs) != 1 {
t.Fatalf("Sent index with %d files, should be 1", len(fs))
}
if fs[0].Name != "foo" {
t.Fatalf(`Sent index with file %v, should be "foo"`, fs[0].Name)
}
received <- fs[0]
return
}
fc.mut.Unlock()
// Scan without ignore patterns with "foo" not existing locally
if err := m.ScanFolder("default"); err != nil {
t.Fatal("Failed scanning:", err)
}
f := <-received
if f.Blocks[0].WeakHash != 103547413 {
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
}
buf := make([]byte, len(payload))
err := m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, payload) {
t.Errorf("%s != %s", buf, payload)
}
payload = []byte("bye")
buf = make([]byte, len(payload))
if err := ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777); err != nil {
t.Fatal(err)
}
err = m.Request(device2, "default", "foo", 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false, buf)
if err == nil {
t.Fatalf("expected failure")
}
select {
case f := <-received:
if f.Blocks[0].WeakHash != 41943361 {
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
}
case <-time.After(time.Second):
t.Fatalf("timed out")
}
}
func setupModelWithConnection() (*Model, *fakeConnection, string) {
tmpDir := createTmpDir()
cfg := defaultCfgWrapper.RawCopy()

71
lib/model/rwfolder.go
View File

@@ -1186,36 +1186,32 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
var file fs.File
var weakHashFinder *weakhash.Finder
if weakhash.Enabled {
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
}
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
if len(hashesToFind) > 0 {
file, err = f.fs.Open(state.file.Name)
if err == nil {
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
}
if len(hashesToFind) > 0 {
file, err = f.fs.Open(state.file.Name)
if err == nil {
weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
}
} else {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
}
} else {
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
}
} else {
l.Debugf("not weak hashing %s. weak hashing disabled", state.file.Name)
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
}
for _, block := range state.blocks {
@@ -1239,7 +1235,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
found, err := weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
if _, err := verifyBuffer(buf, block); err != nil {
if verifyBuffer(buf, block) != nil {
return true
}
@@ -1274,17 +1270,8 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
return false
}
hash, err := verifyBuffer(buf, block)
if err != nil {
if hash != nil {
l.Debugf("Finder block mismatch in %s:%s:%d expected %q got %q", folder, path, index, block.Hash, hash)
err = f.model.finder.Fix(folder, path, index, block.Hash, hash)
if err != nil {
l.Warnln("finder fix:", err)
}
} else {
l.Debugln("Finder failed to verify buffer", err)
}
if err := verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
@@ -1324,22 +1311,22 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
}
}
func verifyBuffer(buf []byte, block protocol.BlockInfo) ([]byte, error) {
func verifyBuffer(buf []byte, block protocol.BlockInfo) error {
if len(buf) != int(block.Size) {
return nil, fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
}
hf := sha256.New()
_, err := hf.Write(buf)
if err != nil {
return nil, err
return err
}
hash := hf.Sum(nil)
if !bytes.Equal(hash, block.Hash) {
return hash, fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
return fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
}
return hash, nil
return nil
}
func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPullerState) {
@@ -1411,7 +1398,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
// Fetch the block, while marking the selected device as in use so that
// leastBusy can select another device when someone else asks.
activity.using(selected)
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
buf, lastError := f.model.requestGlobal(selected.ID, f.folderID, state.file.Name, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
activity.done(selected)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "returned error:", lastError)
@@ -1420,7 +1407,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
// Verify that the received block matches the desired hash, if not
// try pulling it from another device.
_, lastError = verifyBuffer(buf, state.block)
lastError = verifyBuffer(buf, state.block)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, "hash mismatch")
continue

46
lib/model/rwfolder_test.go
View File

@@ -434,52 +434,6 @@ func TestCopierCleanup(t *testing.T) {
}
}
// Make sure that the copier routine hashes the content when asked, and pulls
// if it fails to find the block.
func TestLastResortPulling(t *testing.T) {
// Add a file to index (with the incorrect block representation, as content
// doesn't actually match the block list)
file := setUpFile("empty", []int{0})
m := setUpModel(file)
// Pretend that we are handling a new file of the same content but
// with a different name (causing to copy that particular block)
file.Name = "newfile"
iterFn := func(folder, file string, index int32) bool {
return true
}
f := setUpSendReceiveFolder(m)
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, 1)
finisherChan := make(chan *sharedPullerState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
// Run a single copier routine
go f.copierRoutine(copyChan, pullChan, finisherChan)
f.handleFile(file, copyChan, finisherChan, dbUpdateChan)
// Copier should hash empty file, realise that the region it has read
// doesn't match the hash which was advertised by the block map, fix it
// and ask to pull the block.
<-pullChan
// Verify that it did fix the incorrect hash.
if m.finder.Iterate(folders, blocks[0].Hash, iterFn) {
t.Error("Found unexpected block")
}
if !m.finder.Iterate(folders, scanner.SHA256OfNothing, iterFn) {
t.Error("Expected block not found")
}
(<-finisherChan).fd.Close()
os.Remove(filepath.Join("testdata", fs.TempName("newfile")))
}
func TestDeregisterOnFailInCopy(t *testing.T) {
file := setUpFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
defer os.Remove("testdata/" + fs.TempName("filex"))