lib/model, lib/scanner: Efficient inserts/deletes in the middle of the file

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3527

lib/model/rwfolder_test.go

@@ -7,12 +7,15 @@
 package model
 
 import (
+	"crypto/rand"
+	"io"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -72,6 +75,8 @@ func setUpRwFolder(model *Model) rwFolder {
 			stateTracker: newStateTracker("default"),
 			model:        model,
 		},
+
+		mtimeFS:   fs.NewMtimeFS(db.NewNamespacedKV(model.db, "mtime")),
 		dir:       "testdata",
 		queue:     newJobQueue(),
 		errors:    make(map[string]string),
@@ -199,7 +204,7 @@ func TestCopierFinder(t *testing.T) {
 
 	select {
 	case <-pullChan:
-		t.Fatal("Finisher channel has data to be read")
+		t.Fatal("Pull channel has data to be read")
 	case <-finisherChan:
 		t.Fatal("Finisher channel has data to be read")
 	default:
@@ -240,6 +245,132 @@ func TestCopierFinder(t *testing.T) {
 	os.Remove(tempFile)
 }
 
+func TestWeakHash(t *testing.T) {
+	tempFile := filepath.Join("testdata", defTempNamer.TempName("weakhash"))
+	var shift int64 = 10
+	var size int64 = 1 << 20
+	expectBlocks := int(size / protocol.BlockSize)
+	expectPulls := int(shift / protocol.BlockSize)
+	if shift > 0 {
+		expectPulls++
+	}
+
+	cleanup := func() {
+		for _, path := range []string{tempFile, "testdata/weakhash"} {
+			os.Remove(path)
+		}
+	}
+
+	cleanup()
+	defer cleanup()
+
+	f, err := os.Create("testdata/weakhash")
+	if err != nil {
+		t.Error(err)
+	}
+	defer f.Close()
+	_, err = io.CopyN(f, rand.Reader, size)
+	if err != nil {
+		t.Error(err)
+	}
+	info, err := f.Stat()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Create two files, second file has `shifted` bytes random prefix, yet
+	// both are of the same length, for example:
+	// File 1: abcdefgh
+	// File 2: xyabcdef
+	f.Seek(0, os.SEEK_SET)
+	existing, err := scanner.Blocks(f, protocol.BlockSize, size, nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	f.Seek(0, os.SEEK_SET)
+	remainder := io.LimitReader(f, size-shift)
+	prefix := io.LimitReader(rand.Reader, shift)
+	nf := io.MultiReader(prefix, remainder)
+	desired, err := scanner.Blocks(nf, protocol.BlockSize, size, nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	existingFile := protocol.FileInfo{
+		Name:       "weakhash",
+		Blocks:     existing,
+		Size:       size,
+		ModifiedS:  info.ModTime().Unix(),
+		ModifiedNs: int32(info.ModTime().Nanosecond()),
+	}
+	desiredFile := protocol.FileInfo{
+		Name:      "weakhash",
+		Size:      size,
+		Blocks:    desired,
+		ModifiedS: info.ModTime().Unix() + 1,
+	}
+
+	// Setup the model/pull environment
+	m := setUpModel(existingFile)
+	fo := setUpRwFolder(m)
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState, expectBlocks)
+	finisherChan := make(chan *sharedPullerState, 1)
+
+	// Run a single fetcher routine
+	go fo.copierRoutine(copyChan, pullChan, finisherChan)
+
+	// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
+	fo.handleFile(desiredFile, copyChan, finisherChan)
+
+	var pulls []pullBlockState
+	for len(pulls) < expectBlocks {
+		select {
+		case pull := <-pullChan:
+			pulls = append(pulls, pull)
+		case <-time.After(time.Second):
+			t.Error("timed out")
+		}
+	}
+	finish := <-finisherChan
+
+	select {
+	case <-pullChan:
+		t.Fatal("Pull channel has data to be read")
+	case <-finisherChan:
+		t.Fatal("Finisher channel has data to be read")
+	default:
+	}
+
+	finish.fd.Close()
+	if err := os.Remove(tempFile); err != nil && !os.IsNotExist(err) {
+		t.Error(err)
+	}
+
+	// Test 2 - using weak hash, expectPulls blocks pulled.
+	fo.useWeakHash = true
+	fo.handleFile(desiredFile, copyChan, finisherChan)
+
+	pulls = pulls[:0]
+	for len(pulls) < expectPulls {
+		select {
+		case pull := <-pullChan:
+			pulls = append(pulls, pull)
+		case <-time.After(time.Second):
+			t.Error("timed out")
+		}
+	}
+
+	finish = <-finisherChan
+	finish.fd.Close()
+
+	expectShifted := expectBlocks - expectPulls
+	if finish.copyOriginShifted != expectShifted {
+		t.Errorf("did not copy %d shifted", expectShifted)
+	}
+}
+
 // Test that updating a file removes it's old blocks from the blockmap
 func TestCopierCleanup(t *testing.T) {
 	iterFn := func(folder, file string, index int32) bool {