Refactor out the model into a subpackage
This commit is contained in:
Jakob Borg
65
model/blocks.go
Normal file
65
model/blocks.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
)
|
||||
|
||||
type Block struct {
|
||||
Offset uint64
|
||||
Length uint32
|
||||
Hash []byte
|
||||
}
|
||||
|
||||
// Blocks returns the blockwise hash of the reader.
|
||||
func Blocks(r io.Reader, blocksize int) ([]Block, error) {
|
||||
var blocks []Block
|
||||
var offset uint64
|
||||
for {
|
||||
lr := &io.LimitedReader{r, int64(blocksize)}
|
||||
hf := sha256.New()
|
||||
n, err := io.Copy(hf, lr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
b := Block{
|
||||
Offset: offset,
|
||||
Length: uint32(n),
|
||||
Hash: hf.Sum(nil),
|
||||
}
|
||||
blocks = append(blocks, b)
|
||||
offset += uint64(n)
|
||||
}
|
||||
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// BlockDiff returns lists of common and missing (to transform src into tgt)
|
||||
// blocks. Both block lists must have been created with the same block size.
|
||||
func BlockDiff(src, tgt []Block) (have, need []Block) {
|
||||
if len(tgt) == 0 && len(src) != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if len(tgt) != 0 && len(src) == 0 {
|
||||
// Copy the entire file
|
||||
return nil, tgt
|
||||
}
|
||||
|
||||
for i := range tgt {
|
||||
if i >= len(src) || bytes.Compare(tgt[i].Hash, src[i].Hash) != 0 {
|
||||
// Copy differing block
|
||||
need = append(need, tgt[i])
|
||||
} else {
|
||||
have = append(have, tgt[i])
|
||||
}
|
||||
}
|
||||
|
||||
return have, need
|
||||
}
|
||||
115
model/blocks_test.go
Normal file
115
model/blocks_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var blocksTestData = []struct {
|
||||
data []byte
|
||||
blocksize int
|
||||
hash []string
|
||||
}{
|
||||
{[]byte(""), 1024, []string{}},
|
||||
{[]byte("contents"), 1024, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 9, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 8, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 7, []string{
|
||||
"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89"},
|
||||
},
|
||||
{[]byte("contents"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
|
||||
},
|
||||
{[]byte("conconts"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
|
||||
},
|
||||
{[]byte("contenten"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3"},
|
||||
},
|
||||
}
|
||||
|
||||
func TestBlocks(t *testing.T) {
|
||||
for _, test := range blocksTestData {
|
||||
buf := bytes.NewBuffer(test.data)
|
||||
blocks, err := Blocks(buf, test.blocksize)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if l := len(blocks); l != len(test.hash) {
|
||||
t.Fatalf("Incorrect number of blocks %d != %d", l, len(test.hash))
|
||||
} else {
|
||||
i := 0
|
||||
for off := uint64(0); off < uint64(len(test.data)); off += uint64(test.blocksize) {
|
||||
if blocks[i].Offset != off {
|
||||
t.Errorf("Incorrect offset for block %d: %d != %d", i, blocks[i].Offset, off)
|
||||
}
|
||||
|
||||
bs := test.blocksize
|
||||
if rem := len(test.data) - int(off); bs > rem {
|
||||
bs = rem
|
||||
}
|
||||
if int(blocks[i].Length) != bs {
|
||||
t.Errorf("Incorrect length for block %d: %d != %d", i, blocks[i].Length, bs)
|
||||
}
|
||||
if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
|
||||
t.Errorf("Incorrect block hash %q != %q", h, test.hash[i])
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var diffTestData = []struct {
|
||||
a string
|
||||
b string
|
||||
s int
|
||||
d []Block
|
||||
}{
|
||||
{"contents", "contents", 1024, []Block{}},
|
||||
{"", "", 1024, []Block{}},
|
||||
{"contents", "contents", 3, []Block{}},
|
||||
{"contents", "cantents", 3, []Block{{0, 3, nil}}},
|
||||
{"contents", "contants", 3, []Block{{3, 3, nil}}},
|
||||
{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}},
|
||||
{"contents", "", 3, nil},
|
||||
{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
|
||||
{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
|
||||
{"contents", "con", 3, nil},
|
||||
{"contents", "cont", 3, []Block{{3, 1, nil}}},
|
||||
{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
|
||||
}
|
||||
|
||||
func TestDiff(t *testing.T) {
|
||||
for i, test := range diffTestData {
|
||||
a, _ := Blocks(bytes.NewBufferString(test.a), test.s)
|
||||
b, _ := Blocks(bytes.NewBufferString(test.b), test.s)
|
||||
_, d := BlockDiff(a, b)
|
||||
if len(d) != len(test.d) {
|
||||
t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
|
||||
} else {
|
||||
for j := range test.d {
|
||||
if d[j].Offset != test.d[j].Offset {
|
||||
t.Errorf("Incorrect offset for diff %d block %d; %d != %d", i, j, d[j].Offset, test.d[j].Offset)
|
||||
}
|
||||
if d[j].Length != test.d[j].Length {
|
||||
t.Errorf("Incorrect length for diff %d block %d; %d != %d", i, j, d[j].Length, test.d[j].Length)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
584
model/model.go
Normal file
584
model/model.go
Normal file
@@ -0,0 +1,584 @@
|
||||
package model
|
||||
|
||||
/*
|
||||
|
||||
Locking
|
||||
=======
|
||||
|
||||
The model has read and write locks. These must be acquired as appropriate by
|
||||
public methods. To prevent deadlock situations, private methods should never
|
||||
acquire locks, but document what locks they require.
|
||||
|
||||
*/
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
"github.com/calmh/syncthing/protocol"
|
||||
)
|
||||
|
||||
type Model struct {
|
||||
sync.RWMutex
|
||||
dir string
|
||||
|
||||
global map[string]File // the latest version of each file as it exists in the cluster
|
||||
local map[string]File // the files we currently have locally on disk
|
||||
remote map[string]map[string]File
|
||||
need map[string]bool // the files we need to update
|
||||
nodes map[string]*protocol.Connection
|
||||
rawConn map[string]io.ReadWriteCloser
|
||||
|
||||
updatedLocal int64 // timestamp of last update to local
|
||||
updateGlobal int64 // timestamp of last update to remote
|
||||
|
||||
lastIdxBcast time.Time
|
||||
lastIdxBcastRequest time.Time
|
||||
|
||||
rwRunning bool
|
||||
parallellFiles int
|
||||
paralllelReqs int
|
||||
delete bool
|
||||
|
||||
trace map[string]bool
|
||||
}
|
||||
|
||||
const (
|
||||
FlagDeleted = 1 << 12
|
||||
|
||||
idxBcastHoldtime = 15 * time.Second // Wait at least this long after the last index modification
|
||||
idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
|
||||
)
|
||||
|
||||
// NewModel creates and starts a new model. The model starts in read-only mode,
|
||||
// where it sends index information to connected peers and responds to requests
|
||||
// for file data without altering the local repository in any way.
|
||||
func NewModel(dir string) *Model {
|
||||
m := &Model{
|
||||
dir: dir,
|
||||
global: make(map[string]File),
|
||||
local: make(map[string]File),
|
||||
remote: make(map[string]map[string]File),
|
||||
need: make(map[string]bool),
|
||||
nodes: make(map[string]*protocol.Connection),
|
||||
rawConn: make(map[string]io.ReadWriteCloser),
|
||||
lastIdxBcast: time.Now(),
|
||||
trace: make(map[string]bool),
|
||||
}
|
||||
|
||||
go m.broadcastIndexLoop()
|
||||
return m
|
||||
}
|
||||
|
||||
// Trace enables trace logging of the given facility. This is a debugging function; grep for m.trace.
|
||||
func (m *Model) Trace(t string) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
m.trace[t] = true
|
||||
}
|
||||
|
||||
// StartRW starts read/write processing on the current model. When in
|
||||
// read/write mode the model will attempt to keep in sync with the cluster by
|
||||
// pulling needed files from peer nodes.
|
||||
func (m *Model) StartRW(del bool, pfiles, preqs int) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if m.rwRunning {
|
||||
panic("starting started model")
|
||||
}
|
||||
|
||||
m.rwRunning = true
|
||||
m.delete = del
|
||||
m.parallellFiles = pfiles
|
||||
m.paralllelReqs = preqs
|
||||
|
||||
go m.cleanTempFiles()
|
||||
go m.puller()
|
||||
}
|
||||
|
||||
// Generation returns an opaque integer that is guaranteed to increment on
|
||||
// every change to the local repository or global model.
|
||||
func (m *Model) Generation() int64 {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
return m.updatedLocal + m.updateGlobal
|
||||
}
|
||||
|
||||
type ConnectionInfo struct {
|
||||
protocol.Statistics
|
||||
Address string
|
||||
}
|
||||
|
||||
// ConnectionStats returns a map with connection statistics for each connected node.
|
||||
func (m *Model) ConnectionStats() map[string]ConnectionInfo {
|
||||
type remoteAddrer interface {
|
||||
RemoteAddr() net.Addr
|
||||
}
|
||||
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
var res = make(map[string]ConnectionInfo)
|
||||
for node, conn := range m.nodes {
|
||||
ci := ConnectionInfo{
|
||||
Statistics: conn.Statistics(),
|
||||
}
|
||||
if nc, ok := m.rawConn[node].(remoteAddrer); ok {
|
||||
ci.Address = nc.RemoteAddr().String()
|
||||
}
|
||||
res[node] = ci
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// LocalSize returns the number of files, deleted files and total bytes for all
|
||||
// files in the global model.
|
||||
func (m *Model) GlobalSize() (files, deleted, bytes int) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
for _, f := range m.global {
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
files++
|
||||
bytes += f.Size()
|
||||
} else {
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// LocalSize returns the number of files, deleted files and total bytes for all
|
||||
// files in the local repository.
|
||||
func (m *Model) LocalSize() (files, deleted, bytes int) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
for _, f := range m.local {
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
files++
|
||||
bytes += f.Size()
|
||||
} else {
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// InSyncSize returns the number and total byte size of the local files that
|
||||
// are in sync with the global model.
|
||||
func (m *Model) InSyncSize() (files, bytes int) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
for n, f := range m.local {
|
||||
if gf, ok := m.global[n]; ok && f.Modified == gf.Modified {
|
||||
files++
|
||||
bytes += f.Size()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NeedFiles returns the list of currently needed files and the total size.
|
||||
func (m *Model) NeedFiles() (files []File, bytes int) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
|
||||
for n := range m.need {
|
||||
f := m.global[n]
|
||||
files = append(files, f)
|
||||
bytes += f.Size()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Index is called when a new node is connected and we receive their full index.
|
||||
// Implements the protocol.Model interface.
|
||||
func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if m.trace["net"] {
|
||||
log.Printf("NET IDX(in): %s: %d files", nodeID, len(fs))
|
||||
}
|
||||
|
||||
m.remote[nodeID] = make(map[string]File)
|
||||
for _, f := range fs {
|
||||
m.remote[nodeID][f.Name] = fileFromFileInfo(f)
|
||||
}
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
// IndexUpdate is called for incremental updates to connected nodes' indexes.
|
||||
// Implements the protocol.Model interface.
|
||||
func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if m.trace["net"] {
|
||||
log.Printf("NET IDXUP(in): %s: %d files", nodeID, len(fs))
|
||||
}
|
||||
|
||||
repo, ok := m.remote[nodeID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, f := range fs {
|
||||
if f.Flags&FlagDeleted != 0 && !m.delete {
|
||||
// Files marked as deleted do not even enter the model
|
||||
continue
|
||||
}
|
||||
repo[f.Name] = fileFromFileInfo(f)
|
||||
}
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
// Close removes the peer from the model and closes the underlyign connection if possible.
|
||||
// Implements the protocol.Model interface.
|
||||
func (m *Model) Close(node string, err error) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
conn, ok := m.rawConn[node]
|
||||
if ok {
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
delete(m.remote, node)
|
||||
delete(m.nodes, node)
|
||||
delete(m.rawConn, node)
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
// Request returns the specified data segment by reading it from local disk.
|
||||
// Implements the protocol.Model interface.
|
||||
func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
|
||||
if m.trace["net"] && nodeID != "<local>" {
|
||||
log.Printf("NET REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
|
||||
}
|
||||
fn := path.Join(m.dir, name)
|
||||
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
buf := buffers.Get(int(size))
|
||||
_, err = fd.ReadAt(buf, int64(offset))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// ReplaceLocal replaces the local repository index with the given list of files.
|
||||
func (m *Model) ReplaceLocal(fs []File) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
var updated bool
|
||||
var newLocal = make(map[string]File)
|
||||
|
||||
for _, f := range fs {
|
||||
newLocal[f.Name] = f
|
||||
if ef := m.local[f.Name]; ef.Modified != f.Modified {
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
|
||||
if m.markDeletedLocals(newLocal) {
|
||||
updated = true
|
||||
}
|
||||
|
||||
if len(newLocal) != len(m.local) {
|
||||
updated = true
|
||||
}
|
||||
|
||||
if updated {
|
||||
m.local = newLocal
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
m.updatedLocal = time.Now().Unix()
|
||||
m.lastIdxBcastRequest = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
// SeedLocal replaces the local repository index with the given list of files,
|
||||
// in protocol data types. Does not track deletes, should only be used to seed
|
||||
// the local index from a cache file at startup.
|
||||
func (m *Model) SeedLocal(fs []protocol.FileInfo) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
m.local = make(map[string]File)
|
||||
for _, f := range fs {
|
||||
m.local[f.Name] = fileFromFileInfo(f)
|
||||
}
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
// ConnectedTo returns true if we are connected to the named node.
|
||||
func (m *Model) ConnectedTo(nodeID string) bool {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
_, ok := m.nodes[nodeID]
|
||||
return ok
|
||||
}
|
||||
|
||||
// ProtocolIndex returns the current local index in protocol data types.
|
||||
func (m *Model) ProtocolIndex() []protocol.FileInfo {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
return m.protocolIndex()
|
||||
}
|
||||
|
||||
// RepoID returns a unique ID representing the current repository location.
|
||||
func (m *Model) RepoID() string {
|
||||
return fmt.Sprintf("%x", sha1.Sum([]byte(m.dir)))
|
||||
}
|
||||
|
||||
// AddConnection adds a new peer connection to the model. An initial index will
|
||||
// be sent to the connected peer, thereafter index updates whenever the local
|
||||
// repository changes.
|
||||
func (m *Model) AddConnection(conn io.ReadWriteCloser, nodeID string) {
|
||||
node := protocol.NewConnection(nodeID, conn, conn, m)
|
||||
|
||||
m.Lock()
|
||||
m.nodes[nodeID] = node
|
||||
m.rawConn[nodeID] = conn
|
||||
m.Unlock()
|
||||
|
||||
m.RLock()
|
||||
idx := m.protocolIndex()
|
||||
m.RUnlock()
|
||||
|
||||
go func() {
|
||||
node.Index(idx)
|
||||
}()
|
||||
}
|
||||
|
||||
// protocolIndex returns the current local index in protocol data types.
|
||||
// Must be called with the read lock held.
|
||||
func (m *Model) protocolIndex() []protocol.FileInfo {
|
||||
var index []protocol.FileInfo
|
||||
for _, f := range m.local {
|
||||
mf := fileInfoFromFile(f)
|
||||
if m.trace["idx"] {
|
||||
var flagComment string
|
||||
if mf.Flags&FlagDeleted != 0 {
|
||||
flagComment = " (deleted)"
|
||||
}
|
||||
log.Printf("IDX: %q m=%d f=%o%s (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, len(mf.Blocks))
|
||||
}
|
||||
index = append(index, mf)
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func (m *Model) requestGlobal(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
|
||||
m.RLock()
|
||||
nc, ok := m.nodes[nodeID]
|
||||
m.RUnlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
|
||||
}
|
||||
|
||||
if m.trace["net"] {
|
||||
log.Printf("NET REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
|
||||
}
|
||||
|
||||
return nc.Request(name, offset, size, hash)
|
||||
}
|
||||
|
||||
func (m *Model) broadcastIndexLoop() {
|
||||
for {
|
||||
m.RLock()
|
||||
bcastRequested := m.lastIdxBcastRequest.After(m.lastIdxBcast)
|
||||
holdtimeExceeded := time.Since(m.lastIdxBcastRequest) > idxBcastHoldtime
|
||||
m.RUnlock()
|
||||
|
||||
maxDelayExceeded := time.Since(m.lastIdxBcast) > idxBcastMaxDelay
|
||||
if bcastRequested && (holdtimeExceeded || maxDelayExceeded) {
|
||||
m.Lock()
|
||||
var indexWg sync.WaitGroup
|
||||
indexWg.Add(len(m.nodes))
|
||||
idx := m.protocolIndex()
|
||||
m.lastIdxBcast = time.Now()
|
||||
for _, node := range m.nodes {
|
||||
node := node
|
||||
if m.trace["net"] {
|
||||
log.Printf("NET IDX(out/loop): %s: %d files", node.ID, len(idx))
|
||||
}
|
||||
go func() {
|
||||
node.Index(idx)
|
||||
indexWg.Done()
|
||||
}()
|
||||
}
|
||||
m.Unlock()
|
||||
indexWg.Wait()
|
||||
}
|
||||
time.Sleep(idxBcastHoldtime)
|
||||
}
|
||||
}
|
||||
|
||||
// markDeletedLocals sets the deleted flag on files that have gone missing locally.
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
|
||||
// For every file in the existing local table, check if they are also
|
||||
// present in the new local table. If they are not, check that we already
|
||||
// had the newest version available according to the global table and if so
|
||||
// note the file as having been deleted.
|
||||
var updated bool
|
||||
for n, f := range m.local {
|
||||
if _, ok := newLocal[n]; !ok {
|
||||
if gf := m.global[n]; gf.Modified <= f.Modified {
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
f.Flags = FlagDeleted
|
||||
f.Modified = f.Modified + 1
|
||||
f.Blocks = nil
|
||||
updated = true
|
||||
}
|
||||
newLocal[n] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
func (m *Model) updateLocal(f File) {
|
||||
if ef, ok := m.local[f.Name]; !ok || ef.Modified != f.Modified {
|
||||
m.local[f.Name] = f
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
m.updatedLocal = time.Now().Unix()
|
||||
m.lastIdxBcastRequest = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) recomputeGlobal() {
|
||||
var newGlobal = make(map[string]File)
|
||||
|
||||
for n, f := range m.local {
|
||||
newGlobal[n] = f
|
||||
}
|
||||
|
||||
for _, fs := range m.remote {
|
||||
for n, f := range fs {
|
||||
if cf, ok := newGlobal[n]; !ok || cf.Modified < f.Modified {
|
||||
newGlobal[n] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Figure out if anything actually changed
|
||||
|
||||
var updated bool
|
||||
if len(newGlobal) != len(m.global) {
|
||||
updated = true
|
||||
} else {
|
||||
for n, f0 := range newGlobal {
|
||||
if f1, ok := m.global[n]; !ok || f0.Modified != f1.Modified {
|
||||
updated = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if updated {
|
||||
m.updateGlobal = time.Now().Unix()
|
||||
m.global = newGlobal
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) recomputeNeed() {
|
||||
m.need = make(map[string]bool)
|
||||
for n, f := range m.global {
|
||||
hf, ok := m.local[n]
|
||||
if !ok || f.Modified > hf.Modified {
|
||||
if f.Flags&FlagDeleted != 0 && !m.delete {
|
||||
// Don't want to delete files, so forget this need
|
||||
continue
|
||||
}
|
||||
if f.Flags&FlagDeleted != 0 && !ok {
|
||||
// Don't have the file, so don't need to delete it
|
||||
continue
|
||||
}
|
||||
if m.trace["need"] {
|
||||
log.Println("NEED:", ok, hf, f)
|
||||
}
|
||||
m.need[n] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called with the read lock held.
|
||||
func (m *Model) whoHas(name string) []string {
|
||||
var remote []string
|
||||
|
||||
gf := m.global[name]
|
||||
for node, files := range m.remote {
|
||||
if file, ok := files[name]; ok && file.Modified == gf.Modified {
|
||||
remote = append(remote, node)
|
||||
}
|
||||
}
|
||||
|
||||
return remote
|
||||
}
|
||||
|
||||
func fileFromFileInfo(f protocol.FileInfo) File {
|
||||
var blocks []Block
|
||||
var offset uint64
|
||||
for _, b := range f.Blocks {
|
||||
blocks = append(blocks, Block{
|
||||
Offset: offset,
|
||||
Length: b.Length,
|
||||
Hash: b.Hash,
|
||||
})
|
||||
offset += uint64(b.Length)
|
||||
}
|
||||
return File{
|
||||
Name: f.Name,
|
||||
Flags: f.Flags,
|
||||
Modified: int64(f.Modified),
|
||||
Blocks: blocks,
|
||||
}
|
||||
}
|
||||
|
||||
func fileInfoFromFile(f File) protocol.FileInfo {
|
||||
var blocks []protocol.BlockInfo
|
||||
for _, b := range f.Blocks {
|
||||
blocks = append(blocks, protocol.BlockInfo{
|
||||
Length: b.Length,
|
||||
Hash: b.Hash,
|
||||
})
|
||||
}
|
||||
return protocol.FileInfo{
|
||||
Name: f.Name,
|
||||
Flags: f.Flags,
|
||||
Modified: int64(f.Modified),
|
||||
Blocks: blocks,
|
||||
}
|
||||
}
|
||||
252
model/model_puller.go
Normal file
252
model/model_puller.go
Normal file
@@ -0,0 +1,252 @@
|
||||
package model
|
||||
|
||||
/*
|
||||
|
||||
Locking
|
||||
=======
|
||||
|
||||
These methods are never called from the outside so don't follow the locking
|
||||
policy in model.go.
|
||||
|
||||
TODO(jb): Refactor this into smaller and cleaner pieces.
|
||||
TODO(jb): Increase performance by taking apparent peer bandwidth into account.
|
||||
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
)
|
||||
|
||||
func (m *Model) pullFile(name string) error {
|
||||
m.RLock()
|
||||
var localFile = m.local[name]
|
||||
var globalFile = m.global[name]
|
||||
var nodeIDs = m.whoHas(name)
|
||||
m.RUnlock()
|
||||
|
||||
if len(nodeIDs) == 0 {
|
||||
return fmt.Errorf("%s: no connected nodes with file available", name)
|
||||
}
|
||||
|
||||
filename := path.Join(m.dir, name)
|
||||
sdir := path.Dir(filename)
|
||||
|
||||
_, err := os.Stat(sdir)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
os.MkdirAll(sdir, 0777)
|
||||
}
|
||||
|
||||
tmpFilename := tempName(filename, globalFile.Modified)
|
||||
tmpFile, err := os.Create(tmpFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contentChan := make(chan content, 32)
|
||||
var applyDone sync.WaitGroup
|
||||
applyDone.Add(1)
|
||||
go func() {
|
||||
applyContent(contentChan, tmpFile)
|
||||
tmpFile.Close()
|
||||
applyDone.Done()
|
||||
}()
|
||||
|
||||
local, remote := BlockDiff(localFile.Blocks, globalFile.Blocks)
|
||||
var fetchDone sync.WaitGroup
|
||||
|
||||
// One local copy routine
|
||||
|
||||
fetchDone.Add(1)
|
||||
go func() {
|
||||
for _, block := range local {
|
||||
data, err := m.Request("<local>", name, block.Offset, block.Length, block.Hash)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
contentChan <- content{
|
||||
offset: int64(block.Offset),
|
||||
data: data,
|
||||
}
|
||||
}
|
||||
fetchDone.Done()
|
||||
}()
|
||||
|
||||
// N remote copy routines
|
||||
|
||||
var remoteBlocks = blockIterator{blocks: remote}
|
||||
for i := 0; i < m.paralllelReqs; i++ {
|
||||
curNode := nodeIDs[i%len(nodeIDs)]
|
||||
fetchDone.Add(1)
|
||||
|
||||
go func(nodeID string) {
|
||||
for {
|
||||
block, ok := remoteBlocks.Next()
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
data, err := m.requestGlobal(nodeID, name, block.Offset, block.Length, block.Hash)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
contentChan <- content{
|
||||
offset: int64(block.Offset),
|
||||
data: data,
|
||||
}
|
||||
}
|
||||
fetchDone.Done()
|
||||
}(curNode)
|
||||
}
|
||||
|
||||
fetchDone.Wait()
|
||||
close(contentChan)
|
||||
applyDone.Wait()
|
||||
|
||||
err = hashCheck(tmpFilename, globalFile.Blocks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %s (deleting)", path.Base(name), err.Error())
|
||||
}
|
||||
|
||||
err = os.Chtimes(tmpFilename, time.Unix(globalFile.Modified, 0), time.Unix(globalFile.Modified, 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Rename(tmpFilename, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) puller() {
|
||||
for {
|
||||
time.Sleep(time.Second)
|
||||
|
||||
var ns []string
|
||||
m.RLock()
|
||||
for n := range m.need {
|
||||
ns = append(ns, n)
|
||||
}
|
||||
m.RUnlock()
|
||||
|
||||
if len(ns) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var limiter = make(chan bool, m.parallellFiles)
|
||||
var allDone sync.WaitGroup
|
||||
|
||||
for _, n := range ns {
|
||||
limiter <- true
|
||||
allDone.Add(1)
|
||||
|
||||
go func(n string) {
|
||||
defer func() {
|
||||
allDone.Done()
|
||||
<-limiter
|
||||
}()
|
||||
|
||||
m.RLock()
|
||||
f, ok := m.global[n]
|
||||
m.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
if m.trace["file"] {
|
||||
log.Printf("FILE: Pull %q", n)
|
||||
}
|
||||
err = m.pullFile(n)
|
||||
} else {
|
||||
if m.trace["file"] {
|
||||
log.Printf("FILE: Remove %q", n)
|
||||
}
|
||||
// Cheerfully ignore errors here
|
||||
_ = os.Remove(path.Join(m.dir, n))
|
||||
}
|
||||
if err == nil {
|
||||
m.Lock()
|
||||
m.updateLocal(f)
|
||||
m.Unlock()
|
||||
}
|
||||
}(n)
|
||||
}
|
||||
|
||||
allDone.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
type content struct {
|
||||
offset int64
|
||||
data []byte
|
||||
}
|
||||
|
||||
func applyContent(cc <-chan content, dst io.WriterAt) error {
|
||||
var err error
|
||||
|
||||
for c := range cc {
|
||||
_, err = dst.WriteAt(c.data, c.offset)
|
||||
buffers.Put(c.data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func hashCheck(name string, correct []Block) error {
|
||||
rf, err := os.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rf.Close()
|
||||
|
||||
current, err := Blocks(rf, BlockSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(current) != len(correct) {
|
||||
return errors.New("incorrect number of blocks")
|
||||
}
|
||||
for i := range current {
|
||||
if bytes.Compare(current[i].Hash, correct[i].Hash) != 0 {
|
||||
return fmt.Errorf("hash mismatch: %x != %x", current[i], correct[i])
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type blockIterator struct {
|
||||
sync.Mutex
|
||||
blocks []Block
|
||||
}
|
||||
|
||||
func (i *blockIterator) Next() (b Block, ok bool) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
if len(i.blocks) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
b, i.blocks = i.blocks[0], i.blocks[1:]
|
||||
ok = true
|
||||
|
||||
return
|
||||
}
|
||||
308
model/model_test.go
Normal file
308
model/model_test.go
Normal file
@@ -0,0 +1,308 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/protocol"
|
||||
)
|
||||
|
||||
func TestNewModel(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
|
||||
if m == nil {
|
||||
t.Fatalf("NewModel returned nil")
|
||||
}
|
||||
|
||||
if len(m.need) > 0 {
|
||||
t.Errorf("New model should have no Need")
|
||||
}
|
||||
|
||||
if len(m.local) > 0 {
|
||||
t.Errorf("New model should have no Have")
|
||||
}
|
||||
}
|
||||
|
||||
var testDataExpected = map[string]File{
|
||||
"foo": File{
|
||||
Name: "foo",
|
||||
Flags: 0644,
|
||||
Modified: 1384244572,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
|
||||
},
|
||||
"bar": File{
|
||||
Name: "bar",
|
||||
Flags: 0644,
|
||||
Modified: 1384244579,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
|
||||
},
|
||||
"baz/quux": File{
|
||||
Name: "baz/quux",
|
||||
Flags: 0644,
|
||||
Modified: 1384244676,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0x9, Hash: []uint8{0xc1, 0x54, 0xd9, 0x4e, 0x94, 0xba, 0x72, 0x98, 0xa6, 0xad, 0xb0, 0x52, 0x3a, 0xfe, 0x34, 0xd1, 0xb6, 0xa5, 0x81, 0xd6, 0xb8, 0x93, 0xa7, 0x63, 0xd4, 0x5d, 0xdc, 0x5e, 0x20, 0x9d, 0xcb, 0x83}}},
|
||||
},
|
||||
}
|
||||
|
||||
func TestUpdateLocal(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if len(m.need) > 0 {
|
||||
t.Fatalf("Model with only local data should have no need")
|
||||
}
|
||||
|
||||
if l1, l2 := len(m.local), len(testDataExpected); l1 != l2 {
|
||||
t.Fatalf("Model len(local) incorrect, %d != %d", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(testDataExpected); l1 != l2 {
|
||||
t.Fatalf("Model len(global) incorrect, %d != %d", l1, l2)
|
||||
}
|
||||
for name, file := range testDataExpected {
|
||||
if f, ok := m.local[name]; ok {
|
||||
if !reflect.DeepEqual(f, file) {
|
||||
t.Errorf("Incorrect local\n%v !=\n%v\nfor file %q", f, file, name)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("Missing file %q in local table", name)
|
||||
}
|
||||
if f, ok := m.global[name]; ok {
|
||||
if !reflect.DeepEqual(f, file) {
|
||||
t.Errorf("Incorrect global\n%v !=\n%v\nfor file %q", f, file, name)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("Missing file %q in global table", name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range fs {
|
||||
if hf, ok := m.local[f.Name]; !ok || hf.Modified != f.Modified {
|
||||
t.Fatalf("Incorrect local for %q", f.Name)
|
||||
}
|
||||
if cf, ok := m.global[f.Name]; !ok || cf.Modified != f.Modified {
|
||||
t.Fatalf("Incorrect global for %q", f.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpdateExisting(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "foo",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index("42", []protocol.FileInfo{newFile})
|
||||
|
||||
if l := len(m.need); l != 1 {
|
||||
t.Errorf("Model missing Need for one file (%d != 1)", l)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteAddNew(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "a new file",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index("42", []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.need), 1; l1 != l2 {
|
||||
t.Errorf("Model len(m.need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpdateOld(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
oldTimeStamp := int64(1234)
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "foo",
|
||||
Modified: oldTimeStamp,
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index("42", []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteIndexUpdate(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
foo := protocol.FileInfo{
|
||||
Name: "foo",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
|
||||
bar := protocol.FileInfo{
|
||||
Name: "bar",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
|
||||
m.Index("42", []protocol.FileInfo{foo})
|
||||
|
||||
if _, ok := m.need["foo"]; !ok {
|
||||
t.Error("Model doesn't need 'foo'")
|
||||
}
|
||||
|
||||
m.IndexUpdate("42", []protocol.FileInfo{bar})
|
||||
|
||||
if _, ok := m.need["foo"]; !ok {
|
||||
t.Error("Model doesn't need 'foo'")
|
||||
}
|
||||
if _, ok := m.need["bar"]; !ok {
|
||||
t.Error("Model doesn't need 'bar'")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
ot := time.Now().Unix()
|
||||
newFile := File{
|
||||
Name: "a new file",
|
||||
Modified: ot,
|
||||
Blocks: []Block{{0, 100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.updateLocal(newFile)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
// The deleted file is kept in the local and global tables and marked as deleted.
|
||||
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
if m.local["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in local table")
|
||||
}
|
||||
if len(m.local["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in local")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
if m.global["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in global table")
|
||||
}
|
||||
if len(m.global["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in global")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
// Another update should change nothing
|
||||
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
if m.local["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in local table")
|
||||
}
|
||||
if len(m.local["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in local")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
if m.global["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in global table")
|
||||
}
|
||||
if len(m.global["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in global")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgetNode(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
fs := m.Walk(false)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "new file",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index("42", []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 1; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
m.Close("42", nil)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
||||
1
model/testdata/bar
vendored
Normal file
1
model/testdata/bar
vendored
Normal file
@@ -0,0 +1 @@
|
||||
foobarbaz
|
||||
1
model/testdata/baz/quux
vendored
Normal file
1
model/testdata/baz/quux
vendored
Normal file
@@ -0,0 +1 @@
|
||||
baazquux
|
||||
1
model/testdata/foo
vendored
Normal file
1
model/testdata/foo
vendored
Normal file
@@ -0,0 +1 @@
|
||||
foobar
|
||||
139
model/walk.go
Normal file
139
model/walk.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const BlockSize = 128 * 1024
|
||||
|
||||
type File struct {
|
||||
Name string
|
||||
Flags uint32
|
||||
Modified int64
|
||||
Blocks []Block
|
||||
}
|
||||
|
||||
func (f File) Size() (bytes int) {
|
||||
for _, b := range f.Blocks {
|
||||
bytes += int(b.Length)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func isTempName(name string) bool {
|
||||
return strings.HasPrefix(path.Base(name), ".syncthing.")
|
||||
}
|
||||
|
||||
func tempName(name string, modified int64) string {
|
||||
tdir := path.Dir(name)
|
||||
tname := fmt.Sprintf(".syncthing.%s.%d", path.Base(name), modified)
|
||||
return path.Join(tdir, tname)
|
||||
}
|
||||
|
||||
func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
|
||||
return func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if isTempName(p) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.Mode()&os.ModeType == 0 {
|
||||
rn, err := filepath.Rel(m.dir, p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
fi, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
modified := fi.ModTime().Unix()
|
||||
|
||||
m.RLock()
|
||||
hf, ok := m.local[rn]
|
||||
m.RUnlock()
|
||||
|
||||
if ok && hf.Modified == modified {
|
||||
// No change
|
||||
*res = append(*res, hf)
|
||||
} else {
|
||||
if m.trace["file"] {
|
||||
log.Printf("FILE: Hash %q", p)
|
||||
}
|
||||
fd, err := os.Open(p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
blocks, err := Blocks(fd, BlockSize)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
f := File{
|
||||
Name: rn,
|
||||
Flags: uint32(info.Mode()),
|
||||
Modified: modified,
|
||||
Blocks: blocks,
|
||||
}
|
||||
*res = append(*res, f)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Walk returns the list of files found in the local repository by scanning the
|
||||
// file system. Files are blockwise hashed.
|
||||
func (m *Model) Walk(followSymlinks bool) []File {
|
||||
var files []File
|
||||
fn := m.genWalker(&files)
|
||||
filepath.Walk(m.dir, fn)
|
||||
|
||||
if followSymlinks {
|
||||
d, err := os.Open(m.dir)
|
||||
if err != nil {
|
||||
return files
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
fis, err := d.Readdir(-1)
|
||||
if err != nil {
|
||||
return files
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if fi.Mode()&os.ModeSymlink != 0 {
|
||||
filepath.Walk(path.Join(m.dir, fi.Name())+"/", fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeType == 0 && isTempName(path) {
|
||||
if m.trace["file"] {
|
||||
log.Printf("FILE: Remove %q", path)
|
||||
}
|
||||
os.Remove(path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) cleanTempFiles() {
|
||||
filepath.Walk(m.dir, m.cleanTempFile)
|
||||
}
|
||||
42
model/walk_test.go
Normal file
42
model/walk_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testdata = []struct {
|
||||
name string
|
||||
size int
|
||||
hash string
|
||||
}{
|
||||
{"bar", 10, "2f72cc11a6fcd0271ecef8c61056ee1eb1243be3805bf9a9df98f92f7636b05c"},
|
||||
{"baz/quux", 9, "c154d94e94ba7298a6adb0523afe34d1b6a581d6b893a763d45ddc5e209dcb83"},
|
||||
{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
|
||||
}
|
||||
|
||||
func TestWalk(t *testing.T) {
|
||||
m := NewModel("testdata")
|
||||
files := m.Walk(false)
|
||||
|
||||
if l1, l2 := len(files), len(testdata); l1 != l2 {
|
||||
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
|
||||
}
|
||||
|
||||
for i := range testdata {
|
||||
if n1, n2 := testdata[i].name, files[i].Name; n1 != n2 {
|
||||
t.Errorf("Incorrect file name %q != %q for case #%d", n1, n2, i)
|
||||
}
|
||||
|
||||
if h1, h2 := fmt.Sprintf("%x", files[i].Blocks[0].Hash), testdata[i].hash; h1 != h2 {
|
||||
t.Errorf("Incorrect hash %q != %q for case #%d", h1, h2, i)
|
||||
}
|
||||
|
||||
t0 := time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
t1 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
if mt := files[i].Modified; mt < t0 || mt > t1 {
|
||||
t.Errorf("Unrealistic modtime %d for test %d", mt, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user