lib/db, lib/fs, lib/model: Introduce fs.MtimeFS, remove VirtualMtimeRepo

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3479

lib/fs/mtimefs.go

@@ -0,0 +1,139 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go mtime.proto
+//go:generate protoc --proto_name=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. mtime.proto
+
+package fs
+
+import (
+	"os"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+// The database is where we store the virtual mtimes
+type database interface {
+	Bytes(key string) (data []byte, ok bool)
+	PutBytes(key string, data []byte)
+	Delete(key string)
+}
+
+// variable so that we can mock it for testing
+var osChtimes = os.Chtimes
+
+// The MtimeFS is a filesystem with nanosecond mtime precision, regardless
+// of what shenanigans the underlying filesystem gets up to.
+type MtimeFS struct {
+	db database
+}
+
+func NewMtimeFS(db database) *MtimeFS {
+	return &MtimeFS{
+		db: db,
+	}
+}
+
+func (f *MtimeFS) Chtimes(name string, atime, mtime time.Time) error {
+	// Do a normal Chtimes call, don't care if it succeeds or not.
+	osChtimes(name, atime, mtime)
+
+	// Stat the file to see what happened. Here we *do* return an error,
+	// because it might be "does not exist" or similar. osutil.Lstat is the
+	// souped up version to account for Android breakage.
+	info, err := osutil.Lstat(name)
+	if err != nil {
+		return err
+	}
+
+	f.save(name, info.ModTime(), mtime)
+	return nil
+}
+
+func (f *MtimeFS) Lstat(name string) (os.FileInfo, error) {
+	info, err := osutil.Lstat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	real, virtual := f.load(name)
+	if real == info.ModTime() {
+		info = mtimeFileInfo{
+			FileInfo: info,
+			mtime:    virtual,
+		}
+	}
+
+	return info, nil
+}
+
+// "real" is the on disk timestamp
+// "virtual" is what want the timestamp to be
+
+func (f *MtimeFS) save(name string, real, virtual time.Time) {
+	if real.Equal(virtual) {
+		// If the virtual time and the real on disk time are equal we don't
+		// need to store anything.
+		f.db.Delete(name)
+		return
+	}
+
+	mtime := dbMtime{
+		real:    real,
+		virtual: virtual,
+	}
+	bs, _ := mtime.Marshal() // Can't fail
+	f.db.PutBytes(name, bs)
+}
+
+func (f *MtimeFS) load(name string) (real, virtual time.Time) {
+	data, exists := f.db.Bytes(name)
+	if !exists {
+		return
+	}
+
+	var mtime dbMtime
+	if err := mtime.Unmarshal(data); err != nil {
+		return
+	}
+
+	return mtime.real, mtime.virtual
+}
+
+// The mtimeFileInfo is an os.FileInfo that lies about the ModTime().
+
+type mtimeFileInfo struct {
+	os.FileInfo
+	mtime time.Time
+}
+
+func (m mtimeFileInfo) ModTime() time.Time {
+	return m.mtime
+}
+
+// The dbMtime is our database representation
+
+type dbMtime struct {
+	real    time.Time
+	virtual time.Time
+}
+
+func (t *dbMtime) Marshal() ([]byte, error) {
+	bs0, _ := t.real.MarshalBinary()
+	bs1, _ := t.virtual.MarshalBinary()
+	return append(bs0, bs1...), nil
+}
+
+func (t *dbMtime) Unmarshal(bs []byte) error {
+	if err := t.real.UnmarshalBinary(bs[:len(bs)/2]); err != nil {
+		return err
+	}
+	if err := t.virtual.UnmarshalBinary(bs[len(bs)/2:]); err != nil {
+		return err
+	}
+	return nil
+}

lib/fs/mtimefs_test.go

@@ -0,0 +1,111 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+func TestMtimeFS(t *testing.T) {
+	osutil.RemoveAll("testdata")
+	defer osutil.RemoveAll("testdata")
+	os.Mkdir("testdata", 0755)
+	ioutil.WriteFile("testdata/exists0", []byte("hello"), 0644)
+	ioutil.WriteFile("testdata/exists1", []byte("hello"), 0644)
+	ioutil.WriteFile("testdata/exists2", []byte("hello"), 0644)
+
+	// a random time with nanosecond precision
+	testTime := time.Unix(1234567890, 123456789)
+
+	mtimefs := NewMtimeFS(make(mapStore))
+
+	// Do one Chtimes call that will go through to the normal filesystem
+	osChtimes = os.Chtimes
+	if err := mtimefs.Chtimes("testdata/exists0", testTime, testTime); err != nil {
+		t.Error("Should not have failed:", err)
+	}
+
+	// Do one call that gets an error back from the underlying Chtimes
+	osChtimes = failChtimes
+	if err := mtimefs.Chtimes("testdata/exists1", testTime, testTime); err != nil {
+		t.Error("Should not have failed:", err)
+	}
+
+	// Do one call that gets struck by an exceptionally evil Chtimes
+	osChtimes = evilChtimes
+	if err := mtimefs.Chtimes("testdata/exists2", testTime, testTime); err != nil {
+		t.Error("Should not have failed:", err)
+	}
+
+	// All of the calls were successfull, so an Lstat on them should return
+	// the test timestamp.
+
+	for _, file := range []string{"testdata/exists0", "testdata/exists1", "testdata/exists2"} {
+		if info, err := mtimefs.Lstat(file); err != nil {
+			t.Error("Lstat shouldn't fail:", err)
+		} else if !info.ModTime().Equal(testTime) {
+			t.Errorf("Time mismatch; %v != expected %v", info.ModTime(), testTime)
+		}
+	}
+
+	// The two last files should certainly not have the correct timestamp
+	// when looking directly on disk though.
+
+	for _, file := range []string{"testdata/exists1", "testdata/exists2"} {
+		if info, err := os.Lstat(file); err != nil {
+			t.Error("Lstat shouldn't fail:", err)
+		} else if info.ModTime().Equal(testTime) {
+			t.Errorf("Unexpected time match; %v == %v", info.ModTime(), testTime)
+		}
+	}
+
+	// Changing the timestamp on disk should be reflected in a new Lstat
+	// call. Choose a time that is likely to be able to be on all reasonable
+	// filesystems.
+
+	testTime = time.Now().Add(5 * time.Hour).Truncate(time.Minute)
+	os.Chtimes("testdata/exists0", testTime, testTime)
+	if info, err := mtimefs.Lstat("testdata/exists0"); err != nil {
+		t.Error("Lstat shouldn't fail:", err)
+	} else if !info.ModTime().Equal(testTime) {
+		t.Errorf("Time mismatch; %v != expected %v", info.ModTime(), testTime)
+	}
+}
+
+// The mapStore is a simple database
+
+type mapStore map[string][]byte
+
+func (s mapStore) PutBytes(key string, data []byte) {
+	s[key] = data
+}
+
+func (s mapStore) Bytes(key string) (data []byte, ok bool) {
+	data, ok = s[key]
+	return
+}
+
+func (s mapStore) Delete(key string) {
+	delete(s, key)
+}
+
+// failChtimes does nothing, and fails
+func failChtimes(name string, mtime, atime time.Time) error {
+	return errors.New("no")
+}
+
+// evilChtimes will set an mtime that's 300 days in the future of what was
+// asked for, and truncate the time to the closest hour.
+func evilChtimes(name string, mtime, atime time.Time) error {
+	return os.Chtimes(name, mtime.Add(300*time.Hour).Truncate(time.Hour), atime.Add(300*time.Hour).Truncate(time.Hour))
+}