Virtual mtime support for environments that don't support altering mtimes (fixes #831)

internal/db/leveldb.go

@@ -45,6 +45,7 @@ const (
 	KeyTypeBlock
 	KeyTypeDeviceStatistic
 	KeyTypeFolderStatistic
+	KeyTypeVirtualMtime
 )
 
 type fileVersion struct {
@@ -314,6 +315,8 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) i
 }
 
 func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, myID uint64) int64 {
+	mtimeRepo := NewVirtualMtimeRepo(db, string(folder))
+
 	return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
 		var tf FileInfoTruncated
 		err := tf.UnmarshalXDR(dbi.Value())
@@ -337,6 +340,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.F
 				l.Debugf("batch.Put %p %x", batch, dbi.Key())
 			}
 			batch.Put(dbi.Key(), bs)
+			mtimeRepo.DeleteMtime(tf.Name)
 			ldbUpdateGlobal(db, batch, folder, device, deviceKeyName(dbi.Key()), f.Version)
 			return ts
 		}

internal/db/namespaced.go

@@ -111,6 +111,24 @@ func (n NamespacedKV) String(key string) (string, bool) {
 	return string(valBs), true
 }
 
+// PutBytes stores a new byte slice. Any existing value (even if of another type)
+// is overwritten.
+func (n *NamespacedKV) PutBytes(key string, val []byte) {
+	keyBs := append(n.prefix, []byte(key)...)
+	n.db.Put(keyBs, val, nil)
+}
+
+// Bytes returns the stored value as a raw byte slice and a boolean that
+// is false if no value was stored at the key.
+func (n NamespacedKV) Bytes(key string) ([]byte, bool) {
+	keyBs := append(n.prefix, []byte(key)...)
+	valBs, err := n.db.Get(keyBs, nil)
+	if err != nil {
+		return nil, false
+	}
+	return valBs, true
+}
+
 // Delete deletes the specified key. It is allowed to delete a nonexistent
 // key.
 func (n NamespacedKV) Delete(key string) {

internal/db/set.go

@@ -228,6 +228,7 @@ func DropFolder(db *leveldb.DB, folder string) {
 		folder: folder,
 	}
 	bm.Drop()
+	NewVirtualMtimeRepo(db, folder).Drop()
 }
 
 func normalizeFilenames(fs []protocol.FileInfo) {

internal/db/virtualmtime.go

@@ -0,0 +1,86 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// This type encapsulates a repository of mtimes for platforms where file mtimes
+// can't be set to arbitrary values.  For this to work, we need to store both
+// the mtime we tried to set (the "actual" mtime) as well as the mtime the file
+// has when we're done touching it (the "disk" mtime) so that we can tell if it
+// was changed.  So in GetMtime(), it's not sufficient that the record exists --
+// the argument must also equal the "disk" mtime in the record, otherwise it's
+// been touched locally and the "disk" mtime is actually correct.
+
+type VirtualMtimeRepo struct {
+	ns *NamespacedKV
+}
+
+func NewVirtualMtimeRepo(ldb *leveldb.DB, folder string) *VirtualMtimeRepo {
+	prefix := string(KeyTypeVirtualMtime) + folder
+
+	return &VirtualMtimeRepo{
+		ns: NewNamespacedKV(ldb, prefix),
+	}
+}
+
+func (r *VirtualMtimeRepo) UpdateMtime(path string, diskMtime, actualMtime time.Time) {
+	if debug {
+		l.Debugf("virtual mtime: storing values for path:%s disk:%v actual:%v", path, diskMtime, actualMtime)
+	}
+
+	diskBytes, _ := diskMtime.MarshalBinary()
+	actualBytes, _ := actualMtime.MarshalBinary()
+
+	data := append(diskBytes, actualBytes...)
+
+	r.ns.PutBytes(path, data)
+}
+
+func (r *VirtualMtimeRepo) GetMtime(path string, diskMtime time.Time) time.Time {
+	var debugResult string
+
+	if data, exists := r.ns.Bytes(path); exists {
+		var mtime time.Time
+
+		if err := mtime.UnmarshalBinary(data[:len(data)/2]); err != nil {
+			panic(fmt.Sprintf("Can't unmarshal stored mtime at path %v: %v", path, err))
+		}
+
+		if mtime.Equal(diskMtime) {
+			if err := mtime.UnmarshalBinary(data[len(data)/2:]); err != nil {
+				panic(fmt.Sprintf("Can't unmarshal stored mtime at path %v: %v", path, err))
+			}
+
+			debugResult = "got it"
+			diskMtime = mtime
+		} else if debug {
+			debugResult = fmt.Sprintf("record exists, but mismatch inDisk:%v dbDisk:%v", diskMtime, mtime)
+		}
+	} else {
+		debugResult = "record does not exist"
+	}
+
+	if debug {
+		l.Debugf("virtual mtime: value get result:%v path:%s", debugResult, path)
+	}
+
+	return diskMtime
+}
+
+func (r *VirtualMtimeRepo) DeleteMtime(path string) {
+	r.ns.Delete(path)
+}
+
+func (r *VirtualMtimeRepo) Drop() {
+	r.ns.Reset()
+}

internal/db/virtualmtime_test.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestVirtualMtimeRepo(t *testing.T) {
+	ldb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A few repos so we can ensure they don't pollute each other
+	repo1 := NewVirtualMtimeRepo(ldb, "folder1")
+	repo2 := NewVirtualMtimeRepo(ldb, "folder2")
+
+	// Since GetMtime() returns its argument if the key isn't found or is outdated, we need a dummy to test with.
+	dummyTime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
+
+	// Some times to test with
+	time1 := time.Date(2001, time.February, 3, 4, 5, 7, 0, time.UTC)
+	time2 := time.Date(2010, time.February, 3, 4, 5, 6, 0, time.UTC)
+
+	file1 := "file1.txt"
+
+	// Files are not present at the start
+
+	if v := repo1.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be missing (%v) from repo 1 but it's %v", dummyTime, v)
+	}
+
+	if v := repo2.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be missing (%v) from repo 2 but it's %v", dummyTime, v)
+	}
+
+	repo1.UpdateMtime(file1, time1, time2)
+
+	// Now it should return time2 only when time1 is passed as the argument
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time2) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time2, time1, v)
+	}
+
+	if v := repo1.GetMtime(file1, dummyTime); !v.Equal(dummyTime) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", dummyTime, dummyTime, v)
+	}
+
+	// repo2 shouldn't know about this file
+
+	if v := repo2.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v in repo 2 but we got %v", time1, time1, v)
+	}
+
+	repo1.DeleteMtime(file1)
+
+	// Now it should be gone
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time1, time1, v)
+	}
+
+	// Try again but with Drop()
+
+	repo1.UpdateMtime(file1, time1, time2)
+	repo1.Drop()
+
+	if v := repo1.GetMtime(file1, time1); !v.Equal(time1) {
+		t.Errorf("Mtime should be %v for disk time %v but we got %v", time1, time1, v)
+	}
+}