perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Deprecate scanner.Block & File

Browse Source
This commit is contained in:
Jakob Borg
2014-07-12 23:06:48 +02:00
parent 91b35118d9
commit 655acb4cb2
21 changed files with 269 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
model/blockqueue.go
View File

@@ -7,19 +7,19 @@ package model
import (
"sync"
"github.com/calmh/syncthing/scanner"
"github.com/calmh/syncthing/protocol"
)
type bqAdd struct {
file scanner.File
have []scanner.Block
need []scanner.Block
file protocol.FileInfo
have []protocol.BlockInfo
need []protocol.BlockInfo
}
type bqBlock struct {
file scanner.File
block scanner.Block // get this block from the network
copy []scanner.Block // copy these blocks from the old version of the file
file protocol.FileInfo
block protocol.BlockInfo // get this block from the network
copy []protocol.BlockInfo // copy these blocks from the old version of the file
first bool
last bool
}

101
model/model.go
View File

@@ -162,11 +162,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
var have int64
for _, repo := range m.nodeRepos[node] {
m.repoFiles[repo].WithGlobal(func(f scanner.File) bool {
m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) {
size := f.Size
var size int64
if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize
} else {
size = f.Size()
}
tot += size
have += size
@@ -174,11 +176,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
return true
})
m.repoFiles[repo].WithNeed(node, func(f scanner.File) bool {
m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) {
size := f.Size
var size int64
if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize
} else {
size = f.Size()
}
have -= size
}
@@ -209,7 +213,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
return res
}
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
for _, f := range fs {
fs, de, by := sizeOfFile(f)
files += fs
@@ -219,11 +223,11 @@ func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
return
}
func sizeOfFile(f scanner.File) (files, deleted int, bytes int64) {
func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
if !protocol.IsDeleted(f.Flags) {
files++
if !protocol.IsDirectory(f.Flags) {
bytes += f.Size
bytes += f.Size()
} else {
bytes += zeroEntrySize
}
@@ -240,7 +244,7 @@ func (m *Model) GlobalSize(repo string) (files, deleted int, bytes int64) {
m.rmut.RLock()
defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok {
rf.WithGlobal(func(f scanner.File) bool {
rf.WithGlobal(func(f protocol.FileInfo) bool {
fs, de, by := sizeOfFile(f)
files += fs
deleted += de
@@ -257,7 +261,7 @@ func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) {
m.rmut.RLock()
defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok {
rf.WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
rf.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
fs, de, by := sizeOfFile(f)
files += fs
deleted += de
@@ -275,12 +279,12 @@ func (m *Model) NeedSize(repo string) (files int, bytes int64) {
}
// NeedFiles returns the list of currently needed files
func (m *Model) NeedFilesRepo(repo string) []scanner.File {
func (m *Model) NeedFilesRepo(repo string) []protocol.FileInfo {
m.rmut.RLock()
defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok {
var fs []scanner.File
rf.WithNeed(protocol.LocalNodeID, func(f scanner.File) bool {
var fs []protocol.FileInfo
rf.WithNeed(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
fs = append(fs, f)
return true
})
@@ -304,23 +308,9 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf
return
}
var files = make([]scanner.File, len(fs))
for i := range fs {
f := fs[i]
lamport.Default.Tick(f.Version)
if debug {
var flagComment string
if protocol.IsDeleted(f.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDX(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
files[i] = fileFromFileInfo(f)
}
m.rmut.RLock()
if r, ok := m.repoFiles[repo]; ok {
r.Replace(nodeID, files)
r.Replace(nodeID, fs)
} else {
l.Fatalf("Index for nonexistant repo %q", repo)
}
@@ -339,23 +329,9 @@ func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.F
return
}
var files = make([]scanner.File, len(fs))
for i := range fs {
f := fs[i]
lamport.Default.Tick(f.Version)
if debug {
var flagComment string
if protocol.IsDeleted(f.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
files[i] = fileFromFileInfo(f)
}
m.rmut.RLock()
if r, ok := m.repoFiles[repo]; ok {
r.Update(nodeID, files)
r.Update(nodeID, fs)
} else {
l.Fatalf("IndexUpdate for nonexistant repo %q", repo)
}
@@ -432,14 +408,14 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
}
lf := r.Get(protocol.LocalNodeID, name)
if lf.Suppressed || protocol.IsDeleted(lf.Flags) {
if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
if debug {
l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
}
return nil, ErrInvalid
}
if offset > lf.Size {
if offset > lf.Size() {
if debug {
l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
}
@@ -468,20 +444,20 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
}
// ReplaceLocal replaces the local repository index with the given list of files.
func (m *Model) ReplaceLocal(repo string, fs []scanner.File) {
func (m *Model) ReplaceLocal(repo string, fs []protocol.FileInfo) {
m.rmut.RLock()
m.repoFiles[repo].ReplaceWithDelete(protocol.LocalNodeID, fs)
m.rmut.RUnlock()
}
func (m *Model) CurrentRepoFile(repo string, file string) scanner.File {
func (m *Model) CurrentRepoFile(repo string, file string) protocol.FileInfo {
m.rmut.RLock()
f := m.repoFiles[repo].Get(protocol.LocalNodeID, file)
m.rmut.RUnlock()
return f
}
func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File {
func (m *Model) CurrentGlobalFile(repo string, file string) protocol.FileInfo {
m.rmut.RLock()
f := m.repoFiles[repo].GetGlobal(file)
m.rmut.RUnlock()
@@ -494,7 +470,7 @@ type cFiler struct {
}
// Implements scanner.CurrentFiler
func (cf cFiler) CurrentFile(file string) scanner.File {
func (cf cFiler) CurrentFile(file string) protocol.FileInfo {
return cf.m.CurrentRepoFile(cf.r, file)
}
@@ -552,32 +528,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
// protocolIndex returns the current local index in protocol data types.
func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
var index []protocol.FileInfo
var fs []scanner.File
m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
var fs []protocol.FileInfo
m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
fs = append(fs, f)
return true
})
for _, f := range fs {
mf := fileInfoFromFile(f)
if debug {
var flagComment string
if protocol.IsDeleted(mf.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
}
index = append(index, mf)
}
return index
return fs
}
func (m *Model) updateLocal(repo string, f scanner.File) {
func (m *Model) updateLocal(repo string, f protocol.FileInfo) {
m.rmut.RLock()
m.repoFiles[repo].Update(protocol.LocalNodeID, []scanner.File{f})
m.repoFiles[repo].Update(protocol.LocalNodeID, []protocol.FileInfo{f})
m.rmut.RUnlock()
}
@@ -736,11 +698,10 @@ func (m *Model) LoadIndexes(dir string) {
for repo := range m.repoCfgs {
fs := m.loadIndex(repo, dir)
var sfs = make([]scanner.File, len(fs))
var sfs = make([]protocol.FileInfo, len(fs))
for i := 0; i < len(fs); i++ {
lamport.Default.Tick(fs[i].Version)
sfs[i] = fileFromFileInfo(fs[i])
sfs[i].Suppressed = false // we might have saved an index with files that were suppressed; the should not be on startup
fs[i].Flags &= ^uint32(protocol.FlagInvalid) // we might have saved an index with files that were suppressed; the should not be on startup
}
m.repoFiles[repo].Replace(protocol.LocalNodeID, sfs)

22
model/model_test.go
View File

@@ -13,7 +13,6 @@ import (
"github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
)
@@ -25,27 +24,24 @@ func init() {
node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
}
var testDataExpected = map[string]scanner.File{
"foo": scanner.File{
var testDataExpected = map[string]protocol.FileInfo{
"foo": protocol.FileInfo{
Name: "foo",
Flags: 0,
Modified: 0,
Size: 7,
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
},
"empty": scanner.File{
"empty": protocol.FileInfo{
Name: "empty",
Flags: 0,
Modified: 0,
Size: 0,
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
},
"bar": scanner.File{
"bar": protocol.FileInfo{
Name: "bar",
Flags: 0,
Modified: 0,
Size: 10,
Blocks: []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
},
}
@@ -89,7 +85,7 @@ func genFiles(n int) []protocol.FileInfo {
files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i),
Modified: t,
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
}
}
@@ -212,7 +208,7 @@ func BenchmarkRequest(b *testing.B) {
files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i),
Modified: t,
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
}
}

4
model/puller.go
View File

@@ -21,7 +21,7 @@ import (
type requestResult struct {
node protocol.NodeID
file scanner.File
file protocol.FileInfo
filepath string // full filepath name
offset int64
data []byte
@@ -621,7 +621,7 @@ func (p *puller) queueNeededBlocks() {
}
}
func (p *puller) closeFile(f scanner.File) {
func (p *puller) closeFile(f protocol.FileInfo) {
if debug {
l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
}

46
model/util.go
View File

@@ -6,58 +6,12 @@ package model
import (
"fmt"
"path/filepath"
"sync"
"time"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
func fileFromFileInfo(f protocol.FileInfo) scanner.File {
var blocks = make([]scanner.Block, len(f.Blocks))
var offset int64
for i, b := range f.Blocks {
blocks[i] = scanner.Block{
Offset: offset,
Size: b.Size,
Hash: b.Hash,
}
offset += int64(b.Size)
}
return scanner.File{
// Name is with native separator and normalization
Name: filepath.FromSlash(f.Name),
Size: offset,
Flags: f.Flags &^ protocol.FlagInvalid,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
Suppressed: f.Flags&protocol.FlagInvalid != 0,
}
}
func fileInfoFromFile(f scanner.File) protocol.FileInfo {
var blocks = make([]protocol.BlockInfo, len(f.Blocks))
for i, b := range f.Blocks {
blocks[i] = protocol.BlockInfo{
Size: b.Size,
Hash: b.Hash,
}
}
pf := protocol.FileInfo{
Name: filepath.ToSlash(f.Name),
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
}
if f.Suppressed {
pf.Flags |= protocol.FlagInvalid
}
return pf
}
func cmMap(cm protocol.ClusterConfigMessage) map[string]map[protocol.NodeID]uint32 {
m := make(map[string]map[protocol.NodeID]uint32)
for _, repo := range cm.Repositories {