lib/model, lib/scanner: Efficient inserts/deletes in the middle of the file

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3527

lib/scanner/blocks.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/weakhash"
 )
 
 var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}
@@ -25,6 +26,7 @@ type Counter interface {
 func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
 	hf := sha256.New()
 	hashLength := hf.Size()
+	whf := weakhash.NewHash(blocksize)
 
 	var blocks []protocol.BlockInfo
 	var hashes, thisHash []byte
@@ -44,7 +46,7 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
 	var offset int64
 	for {
 		lr := io.LimitReader(r, int64(blocksize))
-		n, err := copyBuffer(hf, lr, buf)
+		n, err := io.CopyBuffer(hf, io.TeeReader(lr, whf), buf)
 		if err != nil {
 			return nil, err
 		}
@@ -63,15 +65,17 @@ func Blocks(r io.Reader, blocksize int, sizehint int64, counter Counter) ([]prot
 		thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
 
 		b := protocol.BlockInfo{
-			Size:   int32(n),
-			Offset: offset,
-			Hash:   thisHash,
+			Size:     int32(n),
+			Offset:   offset,
+			Hash:     thisHash,
+			WeakHash: whf.Sum32(),
 		}
 
 		blocks = append(blocks, b)
 		offset += int64(n)
 
 		hf.Reset()
+		whf.Reset()
 	}
 
 	if len(blocks) == 0 {
@@ -123,9 +127,12 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo)
 // list and actual reader contents
 func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
 	hf := sha256.New()
+	// A 32k buffer is used for copying into the hash function.
+	buf := make([]byte, 32<<10)
+
 	for i, block := range blocks {
 		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
-		_, err := io.Copy(hf, lr)
+		_, err := io.CopyBuffer(hf, lr, buf)
 		if err != nil {
 			return err
 		}
@@ -180,48 +187,3 @@ func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
 	}
 	return true
 }
-
-// This is a copy & paste of io.copyBuffer from the Go 1.5 standard library,
-// as we want this but also want to build with Go 1.3+.
-
-// copyBuffer is the actual implementation of Copy and CopyBuffer.
-// if buf is nil, one is allocated.
-func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
-	// If the reader has a WriteTo method, use it to do the copy.
-	// Avoids an allocation and a copy.
-	if wt, ok := src.(io.WriterTo); ok {
-		return wt.WriteTo(dst)
-	}
-	// Similarly, if the writer has a ReadFrom method, use it to do the copy.
-	if rt, ok := dst.(io.ReaderFrom); ok {
-		return rt.ReadFrom(src)
-	}
-	if buf == nil {
-		buf = make([]byte, 32*1024)
-	}
-	for {
-		nr, er := src.Read(buf)
-		if nr > 0 {
-			nw, ew := dst.Write(buf[0:nr])
-			if nw > 0 {
-				written += int64(nw)
-			}
-			if ew != nil {
-				err = ew
-				break
-			}
-			if nr != nw {
-				err = io.ErrShortWrite
-				break
-			}
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
-		}
-	}
-	return written, err
-}