perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/protocol, lib/model: Implement high precision time stamps (fixes #3305)

Browse Source 
This adds a new nanoseconds field to the FileInfo, populates it during
scans and sets the non-truncated time in Chtimes calls.

The actual file modification time is defined as modified_s seconds +
modified_ns nanoseconds. It's expected that the modified_ns field is <=
1e9 (that is, all whole seconds should go in the modified_s field) but
not really enforced. Given that it's an int32 the timestamp can be
adjusted += ~2.9 seconds by the modified_ns field...

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3431
This commit is contained in:
Jakob Borg
2016-08-06 13:05:59 +00:00
committed by Audrius Butkevicius
parent 0655991a19
commit ea87bcefd6
21 changed files with 321 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lib/model/model.go
View File

@@ -1658,7 +1658,8 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
Name: f.Name,
Type: f.Type,
Size: f.Size,
Modified: f.Modified,
ModifiedS: f.ModifiedS,
ModifiedNs: f.ModifiedNs,
Permissions: f.Permissions,
NoPermissions: f.NoPermissions,
Invalid: true,
@@ -1676,12 +1677,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
// directory") when we try to Lstat() them.
nf := protocol.FileInfo{
Name: f.Name,
Type: f.Type,
Size: f.Size,
Modified: f.Modified,
Deleted: true,
Version: f.Version.Update(m.shortID),
Name: f.Name,
Type: f.Type,
Size: f.Size,
ModifiedS: f.ModifiedS,
ModifiedNs: f.ModifiedNs,
Deleted: true,
Version: f.Version.Update(m.shortID),
}
batch = append(batch, nf)
@@ -1948,7 +1950,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
if !dirsonly && base != "" {
last[base] = []interface{}{
time.Unix(f.Modified, 0), f.FileSize(),
f.ModTime(), f.FileSize(),
}
}

60
lib/model/model_test.go
View File

@@ -54,22 +54,22 @@ func init() {
var testDataExpected = map[string]protocol.FileInfo{
"foo": {
Name: "foo",
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
Name: "foo",
Type: protocol.FileInfoTypeFile,
ModifiedS: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
},
"empty": {
Name: "empty",
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
Name: "empty",
Type: protocol.FileInfoTypeFile,
ModifiedS: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
},
"bar": {
Name: "bar",
Type: protocol.FileInfoTypeFile,
Modified: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
Name: "bar",
Type: protocol.FileInfoTypeFile,
ModifiedS: 0,
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
},
}
@@ -78,7 +78,7 @@ func init() {
for n, f := range testDataExpected {
fi, _ := os.Stat("testdata/" + n)
f.Permissions = uint32(fi.Mode())
f.Modified = fi.ModTime().Unix()
f.ModifiedS = fi.ModTime().Unix()
f.Size = fi.Size()
testDataExpected[n] = f
}
@@ -144,9 +144,9 @@ func genFiles(n int) []protocol.FileInfo {
t := time.Now().Unix()
for i := 0; i < n; i++ {
files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i),
Modified: t,
Blocks: []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
Name: fmt.Sprintf("file%d", i),
ModifiedS: t,
Blocks: []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
}
}
@@ -284,9 +284,9 @@ func BenchmarkRequest(b *testing.B) {
t := time.Now().Unix()
for i := 0; i < n; i++ {
files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i),
Modified: t,
Blocks: []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
Name: fmt.Sprintf("file%d", i),
ModifiedS: t,
Blocks: []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}},
}
}
@@ -755,11 +755,11 @@ func TestGlobalDirectoryTree(t *testing.T) {
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
}
return protocol.FileInfo{
Name: filepath.Join(path...),
Type: typ,
Modified: 0x666,
Blocks: blocks,
Size: 0xa,
Name: filepath.Join(path...),
Type: typ,
ModifiedS: 0x666,
Blocks: blocks,
Size: 0xa,
}
}
@@ -1006,11 +1006,11 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
}
return protocol.FileInfo{
Name: filepath.Join(path...),
Type: typ,
Modified: 0x666,
Blocks: blocks,
Size: 0xa,
Name: filepath.Join(path...),
Type: typ,
ModifiedS: 0x666,
Blocks: blocks,
Size: 0xa,
}
}
@@ -1148,7 +1148,7 @@ func genDeepFiles(n, d int) []protocol.FileInfo {
i++
}
files[i].Modified = t
files[i].ModifiedS = t
files[i].Blocks = []protocol.BlockInfo{{Offset: 0, Size: 100, Hash: []byte("some hash bytes")}}
}

7
lib/model/queue.go
View File

@@ -9,6 +9,7 @@ package model
import (
"math/rand"
"sort"
"time"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -22,7 +23,7 @@ type jobQueue struct {
type jobQueueEntry struct {
name string
size int64
modified int64
modified time.Time
}
func newJobQueue() *jobQueue {
@@ -31,7 +32,7 @@ func newJobQueue() *jobQueue {
}
}
func (q *jobQueue) Push(file string, size, modified int64) {
func (q *jobQueue) Push(file string, size int64, modified time.Time) {
q.mut.Lock()
q.queued = append(q.queued, jobQueueEntry{file, size, modified})
q.mut.Unlock()
@@ -160,5 +161,5 @@ func (q smallestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] }
type oldestFirst []jobQueueEntry
func (q oldestFirst) Len() int { return len(q) }
func (q oldestFirst) Less(a, b int) bool { return q[a].modified < q[b].modified }
func (q oldestFirst) Less(a, b int) bool { return q[a].modified.Before(q[b].modified) }
func (q oldestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] }

47
lib/model/queue_test.go
View File

@@ -9,6 +9,7 @@ package model
import (
"fmt"
"testing"
"time"
"github.com/d4l3k/messagediff"
)
@@ -16,10 +17,10 @@ import (
func TestJobQueue(t *testing.T) {
// Some random actions
q := newJobQueue()
q.Push("f1", 0, 0)
q.Push("f2", 0, 0)
q.Push("f3", 0, 0)
q.Push("f4", 0, 0)
q.Push("f1", 0, time.Time{})
q.Push("f2", 0, time.Time{})
q.Push("f3", 0, time.Time{})
q.Push("f4", 0, time.Time{})
progress, queued := q.Jobs()
if len(progress) != 0 || len(queued) != 4 {
@@ -44,7 +45,7 @@ func TestJobQueue(t *testing.T) {
t.Fatal("Wrong length", len(progress), len(queued))
}
q.Push(n, 0, 0)
q.Push(n, 0, time.Time{})
progress, queued = q.Jobs()
if len(progress) != 0 || len(queued) != 4 {
t.Fatal("Wrong length")
@@ -121,10 +122,10 @@ func TestJobQueue(t *testing.T) {
func TestBringToFront(t *testing.T) {
q := newJobQueue()
q.Push("f1", 0, 0)
q.Push("f2", 0, 0)
q.Push("f3", 0, 0)
q.Push("f4", 0, 0)
q.Push("f1", 0, time.Time{})
q.Push("f2", 0, time.Time{})
q.Push("f3", 0, time.Time{})
q.Push("f4", 0, time.Time{})
_, queued := q.Jobs()
if diff, equal := messagediff.PrettyDiff([]string{"f1", "f2", "f3", "f4"}, queued); !equal {
@@ -162,10 +163,10 @@ func TestBringToFront(t *testing.T) {
func TestShuffle(t *testing.T) {
q := newJobQueue()
q.Push("f1", 0, 0)
q.Push("f2", 0, 0)
q.Push("f3", 0, 0)
q.Push("f4", 0, 0)
q.Push("f1", 0, time.Time{})
q.Push("f2", 0, time.Time{})
q.Push("f3", 0, time.Time{})
q.Push("f4", 0, time.Time{})
// This test will fail once in eight million times (1 / (4!)^5) :)
for i := 0; i < 5; i++ {
@@ -187,10 +188,10 @@ func TestShuffle(t *testing.T) {
func TestSortBySize(t *testing.T) {
q := newJobQueue()
q.Push("f1", 20, 0)
q.Push("f2", 40, 0)
q.Push("f3", 30, 0)
q.Push("f4", 10, 0)
q.Push("f1", 20, time.Time{})
q.Push("f2", 40, time.Time{})
q.Push("f3", 30, time.Time{})
q.Push("f4", 10, time.Time{})
q.SortSmallestFirst()
@@ -219,10 +220,10 @@ func TestSortBySize(t *testing.T) {
func TestSortByAge(t *testing.T) {
q := newJobQueue()
q.Push("f1", 0, 20)
q.Push("f2", 0, 40)
q.Push("f3", 0, 30)
q.Push("f4", 0, 10)
q.Push("f1", 0, time.Unix(20, 0))
q.Push("f2", 0, time.Unix(40, 0))
q.Push("f3", 0, time.Unix(30, 0))
q.Push("f4", 0, time.Unix(10, 0))
q.SortOldestFirst()
@@ -254,7 +255,7 @@ func BenchmarkJobQueueBump(b *testing.B) {
q := newJobQueue()
for _, f := range files {
q.Push(f.Name, 0, 0)
q.Push(f.Name, 0, time.Time{})
}
b.ResetTimer()
@@ -270,7 +271,7 @@ func BenchmarkJobQueuePushPopDone10k(b *testing.B) {
for i := 0; i < b.N; i++ {
q := newJobQueue()
for _, f := range files {
q.Push(f.Name, 0, 0)
q.Push(f.Name, 0, time.Time{})
}
for _ = range files {
n, _ := q.Pop()

10
lib/model/rwfolder.go
View File

@@ -449,7 +449,7 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
devices := folderFiles.Availability(file.Name)
for _, dev := range devices {
if f.model.ConnectedTo(dev) {
f.queue.Push(file.Name, file.Size, file.Modified)
f.queue.Push(file.Name, file.Size, file.ModTime())
changed++
break
}
@@ -925,7 +925,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
// changes that we don't know about yet and we should scan before
// touching the file. If we can't stat the file we'll just pull it.
if info, err := f.mtimeFS.Lstat(realName); err == nil {
if info.ModTime().Unix() != curFile.Modified || info.Size() != curFile.Size {
if !info.ModTime().Equal(curFile.ModTime()) || info.Size() != curFile.Size {
l.Debugln("file modified but not rescanned; not pulling:", realName)
// Scan() is synchronous (i.e. blocks until the scan is
// completed and returns an error), but a scan can't happen
@@ -1044,8 +1044,7 @@ func (f *rwFolder) shortcutFile(file protocol.FileInfo) error {
}
}
t := time.Unix(file.Modified, 0)
f.mtimeFS.Chtimes(realName, t, t) // never fails
f.mtimeFS.Chtimes(realName, file.ModTime(), file.ModTime()) // never fails
// This may have been a conflict. We should merge the version vectors so
// that our clock doesn't move backwards.
@@ -1247,8 +1246,7 @@ func (f *rwFolder) performFinish(state *sharedPullerState) error {
}
// Set the correct timestamp on the new file
t := time.Unix(state.file.Modified, 0)
f.mtimeFS.Chtimes(state.tempName, t, t) // never fails
f.mtimeFS.Chtimes(state.tempName, state.file.ModTime(), state.file.ModTime()) // never fails
if stat, err := f.mtimeFS.Lstat(state.realName); err == nil {
// There is an old file or directory already in place. We need to

4
lib/model/rwfolder_test.go
View File

@@ -332,7 +332,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
f := setUpRwFolder(m)
// queue.Done should be called by the finisher routine
f.queue.Push("filex", 0, 0)
f.queue.Push("filex", 0, time.Time{})
f.queue.Pop()
if f.queue.lenProgress() != 1 {
@@ -405,7 +405,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
f := setUpRwFolder(m)
// queue.Done should be called by the finisher routine
f.queue.Push("filex", 0, 0)
f.queue.Push("filex", 0, time.Time{})
f.queue.Pop()
if f.queue.lenProgress() != 1 {

3
lib/model/sorter_test.go
View File

@@ -103,7 +103,8 @@ func addFiles(n int, s IndexSorter) {
Name: fmt.Sprintf("file-%d", rnd),
Size: rand.Int63(),
Permissions: uint32(rand.Intn(0777)),
Modified: rand.Int63(),
ModifiedS: rand.Int63(),
ModifiedNs: int32(rand.Int63()),
Sequence: rnd,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: uint64(rand.Int63())}}},
Blocks: []protocol.BlockInfo{{