perso/syncthing-arm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Deprecate scanner.Block & File

Browse Source
This commit is contained in:
Jakob Borg
2014-07-12 23:06:48 +02:00
parent 91b35118d9
commit 655acb4cb2
21 changed files with 269 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
scanner/blocks.go
View File

@@ -7,25 +7,16 @@ package scanner
import (
"bytes"
"crypto/sha256"
"fmt"
"io"
"github.com/calmh/syncthing/protocol"
)
const StandardBlockSize = 128 * 1024
type Block struct {
Offset int64
Size uint32
Hash []byte
}
func (b Block) String() string {
return fmt.Sprintf("%d/%d/%x", b.Offset, b.Size, b.Hash)
}
// Blocks returns the blockwise hash of the reader.
func Blocks(r io.Reader, blocksize int) ([]Block, error) {
var blocks []Block
func Blocks(r io.Reader, blocksize int) ([]protocol.BlockInfo, error) {
var blocks []protocol.BlockInfo
var offset int64
for {
lr := &io.LimitedReader{R: r, N: int64(blocksize)}
@@ -39,9 +30,9 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
break
}
b := Block{
Offset: offset,
b := protocol.BlockInfo{
Size: uint32(n),
Offset: offset,
Hash: hf.Sum(nil),
}
blocks = append(blocks, b)
@@ -50,7 +41,7 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
if len(blocks) == 0 {
// Empty file
blocks = append(blocks, Block{
blocks = append(blocks, protocol.BlockInfo{
Offset: 0,
Size: 0,
Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55},
@@ -62,11 +53,18 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
// BlockDiff returns lists of common and missing (to transform src into tgt)
// blocks. Both block lists must have been created with the same block size.
func BlockDiff(src, tgt []Block) (have, need []Block) {
func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
if len(tgt) == 0 && len(src) != 0 {
return nil, nil
}
// Set the Offset field on each target block
var offset int64
for i := range tgt {
tgt[i].Offset = offset
offset += int64(tgt[i].Size)
}
if len(tgt) != 0 && len(src) == 0 {
// Copy the entire file
return nil, tgt

26
scanner/blocks_test.go
View File

@@ -8,6 +8,8 @@ import (
"bytes"
"fmt"
"testing"
"github.com/calmh/syncthing/protocol"
)
var blocksTestData = []struct {
@@ -83,20 +85,20 @@ var diffTestData = []struct {
a string
b string
s int
d []Block
d []protocol.BlockInfo
}{
{"contents", "contents", 1024, []Block{}},
{"", "", 1024, []Block{}},
{"contents", "contents", 3, []Block{}},
{"contents", "cantents", 3, []Block{{0, 3, nil}}},
{"contents", "contants", 3, []Block{{3, 3, nil}}},
{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}},
{"contents", "", 3, []Block{{0, 0, nil}}},
{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
{"contents", "contents", 1024, []protocol.BlockInfo{}},
{"", "", 1024, []protocol.BlockInfo{}},
{"contents", "contents", 3, []protocol.BlockInfo{}},
{"contents", "cantents", 3, []protocol.BlockInfo{{0, 3, nil}}},
{"contents", "contants", 3, []protocol.BlockInfo{{3, 3, nil}}},
{"contents", "cantants", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}}},
{"contents", "", 3, []protocol.BlockInfo{{0, 0, nil}}},
{"", "contents", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
{"con", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
{"contents", "con", 3, nil},
{"contents", "cont", 3, []Block{{3, 1, nil}}},
{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
{"contents", "cont", 3, []protocol.BlockInfo{{3, 1, nil}}},
{"cont", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
}
func TestDiff(t *testing.T) {

49
scanner/blocks_xdr.go
View File

@@ -1,49 +0,0 @@
package scanner
import (
"bytes"
"io"
"github.com/calmh/syncthing/xdr"
)
func (o Block) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o Block) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o Block) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o Block) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteUint64(uint64(o.Offset))
xw.WriteUint32(o.Size)
xw.WriteBytes(o.Hash)
return xw.Tot(), xw.Error()
}
func (o *Block) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *Block) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *Block) decodeXDR(xr *xdr.Reader) error {
o.Offset = int64(xr.ReadUint64())
o.Size = xr.ReadUint32()
o.Hash = xr.ReadBytes()
return xr.Error()
}

30
scanner/file.go
View File

@@ -1,30 +0,0 @@
// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.
package scanner
import "fmt"
type File struct {
Name string
Flags uint32
Modified int64
Version uint64
Size int64
Blocks []Block
Suppressed bool
}
func (f File) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v, Sup:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size, f.Blocks, f.Suppressed)
}
func (f File) Equals(o File) bool {
return f.Modified == o.Modified && f.Version == o.Version
}
func (f File) NewerThan(o File) bool {
return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
}

64
scanner/file_xdr.go
View File

@@ -1,64 +0,0 @@
package scanner
import (
"bytes"
"io"
"github.com/calmh/syncthing/xdr"
)
func (o File) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o File) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o File) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o File) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteString(o.Name)
xw.WriteUint32(o.Flags)
xw.WriteUint64(uint64(o.Modified))
xw.WriteUint64(o.Version)
xw.WriteUint64(uint64(o.Size))
xw.WriteUint32(uint32(len(o.Blocks)))
for i := range o.Blocks {
o.Blocks[i].encodeXDR(xw)
}
xw.WriteBool(o.Suppressed)
return xw.Tot(), xw.Error()
}
func (o *File) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *File) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *File) decodeXDR(xr *xdr.Reader) error {
o.Name = xr.ReadString()
o.Flags = xr.ReadUint32()
o.Modified = int64(xr.ReadUint64())
o.Version = xr.ReadUint64()
o.Size = int64(xr.ReadUint64())
_BlocksSize := int(xr.ReadUint32())
o.Blocks = make([]Block, _BlocksSize)
for i := range o.Blocks {
(&o.Blocks[i]).decodeXDR(xr)
}
o.Suppressed = xr.ReadBool()
return xr.Error()
}

15
scanner/walk.go
View File

@@ -55,12 +55,12 @@ type Suppressor interface {
type CurrentFiler interface {
// CurrentFile returns the file as seen at last scan.
CurrentFile(name string) File
CurrentFile(name string) protocol.FileInfo
}
// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
func (w *Walker) Walk() (files []protocol.FileInfo, ignore map[string][]string, err error) {
if debug {
l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
}
@@ -122,7 +122,7 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W
}
}
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
func (w *Walker) walkAndHashFiles(res *[]protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
if debug {
@@ -183,7 +183,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
} else {
flags |= uint32(info.Mode() & os.ModePerm)
}
f := File{
f := protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: flags,
@@ -213,8 +213,8 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
if w.Suppressor != nil {
if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
cf.Suppressed = true
cf.Version++
cf.Flags |= protocol.FlagInvalid
cf.Version = lamport.Default.Tick(cf.Version)
if debug {
l.Debugln("suppressed:", cf)
}
@@ -256,10 +256,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
if w.IgnorePerms {
flags = protocol.FlagNoPermBits | 0666
}
f := File{
f := protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Size: info.Size(),
Flags: flags,
Modified: info.ModTime().Unix(),
Blocks: blocks,