lib/scanner: Refactoring
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4642 LGTM: imsodin, AudriusButkevicius
This commit is contained in:
Lars K.W. Gohlke
committed by
Audrius Butkevicius
Audrius Butkevicius
parent
bbc178ccc4
commit
89a021609b
@@ -7,9 +7,7 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
@@ -26,18 +24,20 @@ type Counter interface {
|
||||
|
||||
// Blocks returns the blockwise hash of the reader.
|
||||
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
|
||||
if counter == nil {
|
||||
counter = &noopCounter{}
|
||||
}
|
||||
|
||||
hf := sha256.New()
|
||||
hashLength := hf.Size()
|
||||
|
||||
var mhf io.Writer
|
||||
var whf hash.Hash32
|
||||
|
||||
var weakHf hash.Hash32 = noopHash{}
|
||||
var multiHf io.Writer = hf
|
||||
if useWeakHashes {
|
||||
whf = adler32.New()
|
||||
mhf = io.MultiWriter(hf, whf)
|
||||
} else {
|
||||
whf = noopHash{}
|
||||
mhf = hf
|
||||
// Use an actual weak hash function, make the multiHf
|
||||
// write to both hash functions.
|
||||
weakHf = adler32.New()
|
||||
multiHf = io.MultiWriter(hf, weakHf)
|
||||
}
|
||||
|
||||
var blocks []protocol.BlockInfo
|
||||
@@ -65,7 +65,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
}
|
||||
|
||||
lr.N = int64(blocksize)
|
||||
n, err := io.CopyBuffer(mhf, lr, buf)
|
||||
n, err := io.CopyBuffer(multiHf, lr, buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -74,9 +74,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
break
|
||||
}
|
||||
|
||||
if counter != nil {
|
||||
counter.Update(n)
|
||||
}
|
||||
counter.Update(n)
|
||||
|
||||
// Carve out a hash-sized chunk of "hashes" to store the hash for this
|
||||
// block.
|
||||
@@ -87,14 +85,14 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
Size: int32(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
WeakHash: whf.Sum32(),
|
||||
WeakHash: weakHf.Sum32(),
|
||||
}
|
||||
|
||||
blocks = append(blocks, b)
|
||||
offset += n
|
||||
|
||||
hf.Reset()
|
||||
whf.Reset()
|
||||
weakHf.Reset()
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
@@ -109,104 +107,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// PopulateOffsets sets the Offset field on each block
|
||||
func PopulateOffsets(blocks []protocol.BlockInfo) {
|
||||
var offset int64
|
||||
for i := range blocks {
|
||||
blocks[i].Offset = offset
|
||||
offset += int64(blocks[i].Size)
|
||||
}
|
||||
}
|
||||
|
||||
// BlockDiff returns lists of common and missing (to transform src into tgt)
|
||||
// blocks. Both block lists must have been created with the same block size.
|
||||
func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
|
||||
if len(tgt) == 0 && len(src) != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if len(tgt) != 0 && len(src) == 0 {
|
||||
// Copy the entire file
|
||||
return nil, tgt
|
||||
}
|
||||
|
||||
for i := range tgt {
|
||||
if i >= len(src) || !bytes.Equal(tgt[i].Hash, src[i].Hash) {
|
||||
// Copy differing block
|
||||
need = append(need, tgt[i])
|
||||
} else {
|
||||
have = append(have, tgt[i])
|
||||
}
|
||||
}
|
||||
|
||||
return have, need
|
||||
}
|
||||
|
||||
// Verify returns nil or an error describing the mismatch between the block
|
||||
// list and actual reader contents
|
||||
func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
|
||||
hf := sha256.New()
|
||||
// A 32k buffer is used for copying into the hash function.
|
||||
buf := make([]byte, 32<<10)
|
||||
|
||||
for i, block := range blocks {
|
||||
lr := &io.LimitedReader{R: r, N: int64(blocksize)}
|
||||
_, err := io.CopyBuffer(hf, lr, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hash := hf.Sum(nil)
|
||||
hf.Reset()
|
||||
|
||||
if !bytes.Equal(hash, block.Hash) {
|
||||
return fmt.Errorf("hash mismatch %x != %x for block %d", hash, block.Hash, i)
|
||||
}
|
||||
}
|
||||
|
||||
// We should have reached the end now
|
||||
bs := make([]byte, 1)
|
||||
n, err := r.Read(bs)
|
||||
if n != 0 || err != io.EOF {
|
||||
return fmt.Errorf("file continues past end of blocks")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func VerifyBuffer(buf []byte, block protocol.BlockInfo) ([]byte, error) {
|
||||
if len(buf) != int(block.Size) {
|
||||
return nil, fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
|
||||
}
|
||||
hf := sha256.New()
|
||||
_, err := hf.Write(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash := hf.Sum(nil)
|
||||
|
||||
if !bytes.Equal(hash, block.Hash) {
|
||||
return hash, fmt.Errorf("hash mismatch %x != %x", hash, block.Hash)
|
||||
}
|
||||
|
||||
return hash, nil
|
||||
}
|
||||
|
||||
// BlocksEqual returns whether two slices of blocks are exactly the same hash
|
||||
// and index pair wise.
|
||||
func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
|
||||
if len(tgt) != len(src) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i, sblk := range src {
|
||||
if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type noopHash struct{}
|
||||
|
||||
func (noopHash) Sum32() uint32 { return 0 }
|
||||
@@ -215,3 +115,7 @@ func (noopHash) Size() int { return 0 }
|
||||
func (noopHash) Reset() {}
|
||||
func (noopHash) Sum([]byte) []byte { return nil }
|
||||
func (noopHash) Write([]byte) (int, error) { return 0, nil }
|
||||
|
||||
type noopCounter struct{}
|
||||
|
||||
func (c *noopCounter) Update(bytes int64) {}
|
||||
|
||||
Reference in New Issue
Block a user