Consolidate cmds in cmd/

This commit is contained in:
Jakob Borg
2014-07-23 08:31:13 +02:00
parent 978f68b744
commit 08ca9f9378
6 changed files with 3 additions and 3 deletions

1
cmd/discosrv/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
discosrv

288
cmd/discosrv/main.go Normal file
View File

@@ -0,0 +1,288 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package main
import (
"encoding/binary"
"encoding/hex"
"flag"
"fmt"
"io"
"log"
"net"
"os"
"sync"
"time"
"github.com/calmh/syncthing/discover"
"github.com/calmh/syncthing/protocol"
"github.com/golang/groupcache/lru"
"github.com/juju/ratelimit"
)
type node struct {
addresses []address
updated time.Time
}
type address struct {
ip []byte
port uint16
}
var (
nodes = make(map[protocol.NodeID]node)
lock sync.Mutex
queries = 0
announces = 0
answered = 0
limited = 0
unknowns = 0
debug = false
lruSize = 1024
limitAvg = 1
limitBurst = 10
limiter *lru.Cache
)
func main() {
var listen string
var timestamp bool
var statsIntv int
var statsFile string
flag.StringVar(&listen, "listen", ":22026", "Listen address")
flag.BoolVar(&debug, "debug", false, "Enable debug output")
flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
flag.StringVar(&statsFile, "stats-file", "/var/log/discosrv.stats", "Statistics file name")
flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
flag.Parse()
limiter = lru.New(lruSize)
log.SetOutput(os.Stdout)
if !timestamp {
log.SetFlags(0)
}
addr, _ := net.ResolveUDPAddr("udp", listen)
conn, err := net.ListenUDP("udp", addr)
if err != nil {
log.Fatal(err)
}
if statsIntv > 0 {
go logStats(statsFile, statsIntv)
}
var buf = make([]byte, 1024)
for {
buf = buf[:cap(buf)]
n, addr, err := conn.ReadFromUDP(buf)
if limit(addr) {
// Rate limit in effect for source
continue
}
if err != nil {
log.Fatal(err)
}
if n < 4 {
log.Printf("Received short packet (%d bytes)", n)
continue
}
buf = buf[:n]
magic := binary.BigEndian.Uint32(buf)
switch magic {
case discover.AnnouncementMagic:
handleAnnounceV2(addr, buf)
case discover.QueryMagic:
handleQueryV2(conn, addr, buf)
default:
lock.Lock()
unknowns++
lock.Unlock()
}
}
}
func limit(addr *net.UDPAddr) bool {
key := addr.IP.String()
lock.Lock()
defer lock.Unlock()
bkt, ok := limiter.Get(key)
if ok {
bkt := bkt.(*ratelimit.Bucket)
if bkt.TakeAvailable(1) != 1 {
// Rate limit exceeded; ignore packet
if debug {
log.Println("Rate limit exceeded for", key)
}
limited++
return true
}
} else {
if debug {
log.Println("New limiter for", key)
}
// One packet per ten seconds average rate, burst ten packets
limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
}
return false
}
func handleAnnounceV2(addr *net.UDPAddr, buf []byte) {
var pkt discover.Announce
err := pkt.UnmarshalXDR(buf)
if err != nil && err != io.EOF {
log.Println("AnnounceV2 Unmarshal:", err)
log.Println(hex.Dump(buf))
return
}
if debug {
log.Printf("<- %v %#v", addr, pkt)
}
lock.Lock()
announces++
lock.Unlock()
ip := addr.IP.To4()
if ip == nil {
ip = addr.IP.To16()
}
var addrs []address
for _, addr := range pkt.This.Addresses {
tip := addr.IP
if len(tip) == 0 {
tip = ip
}
addrs = append(addrs, address{
ip: tip,
port: addr.Port,
})
}
node := node{
addresses: addrs,
updated: time.Now(),
}
var id protocol.NodeID
if len(pkt.This.ID) == 32 {
// Raw node ID
copy(id[:], pkt.This.ID)
} else {
id.UnmarshalText(pkt.This.ID)
}
lock.Lock()
nodes[id] = node
lock.Unlock()
}
func handleQueryV2(conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
var pkt discover.Query
err := pkt.UnmarshalXDR(buf)
if err != nil {
log.Println("QueryV2 Unmarshal:", err)
log.Println(hex.Dump(buf))
return
}
if debug {
log.Printf("<- %v %#v", addr, pkt)
}
var id protocol.NodeID
if len(pkt.NodeID) == 32 {
// Raw node ID
copy(id[:], pkt.NodeID)
} else {
id.UnmarshalText(pkt.NodeID)
}
lock.Lock()
node, ok := nodes[id]
queries++
lock.Unlock()
if ok && len(node.addresses) > 0 {
ann := discover.Announce{
Magic: discover.AnnouncementMagic,
This: discover.Node{
ID: pkt.NodeID,
},
}
for _, addr := range node.addresses {
ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
}
if debug {
log.Printf("-> %v %#v", addr, pkt)
}
tb := ann.MarshalXDR()
_, _, err = conn.WriteMsgUDP(tb, nil, addr)
if err != nil {
log.Println("QueryV2 response write:", err)
}
lock.Lock()
answered++
lock.Unlock()
}
}
func next(intv int) time.Time {
d := time.Duration(intv) * time.Second
t0 := time.Now()
t1 := t0.Add(d).Truncate(d)
time.Sleep(t1.Sub(t0))
return t1
}
func logStats(file string, intv int) {
f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Fatal(err)
}
for {
t := next(intv)
lock.Lock()
var deleted = 0
for id, node := range nodes {
if time.Since(node.updated) > 60*time.Minute {
delete(nodes, id)
deleted++
}
}
fmt.Fprintf(f, "%d Nr:%d Ne:%d Qt:%d Qa:%d A:%d U:%d Lq:%d Lc:%d\n",
t.Unix(), len(nodes), deleted, queries, answered, announces, unknowns, limited, limiter.Len())
f.Sync()
queries = 0
announces = 0
answered = 0
limited = 0
unknowns = 0
lock.Unlock()
}
}

431
cmd/genxdr/main.go Normal file
View File

@@ -0,0 +1,431 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"flag"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"os"
"regexp"
"strconv"
"strings"
"text/template"
)
type fieldInfo struct {
Name string
IsBasic bool // handled by one the native Read/WriteUint64 etc functions
IsSlice bool // field is a slice of FieldType
FieldType string // original type of field, i.e. "int"
Encoder string // the encoder name, i.e. "Uint64" for Read/WriteUint64
Convert string // what to convert to when encoding, i.e. "uint64"
Max int // max size for slices and strings
}
type structInfo struct {
Name string
Fields []fieldInfo
}
var headerTpl = template.Must(template.New("header").Parse(`// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package {{.Package}}
import (
"bytes"
"io"
"github.com/calmh/syncthing/xdr"
)
`))
var encodeTpl = template.Must(template.New("encoder").Parse(`
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}//+n
func (o {{.TypeName}}) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}//+n
func (o {{.TypeName}}) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}//+n
func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ErrElementSizeExceeded
}
{{end}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
{{else}}
o.{{$fieldInfo.Name}}.encodeXDR(xw)
{{end}}
{{else}}
{{if ge $fieldInfo.Max 1}}
if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ErrElementSizeExceeded
}
{{end}}
xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
{{else if $fieldInfo.IsBasic}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
{{else}}
o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
{{end}}
}
{{end}}
{{end}}
return xw.Tot(), xw.Error()
}//+n
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}//+n
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}//+n
func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
{{else}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
{{end}}
{{else}}
(&o.{{$fieldInfo.Name}}).decodeXDR(xr)
{{end}}
{{else}}
_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
{{if ge $fieldInfo.Max 1}}
if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
return xdr.ErrElementSizeExceeded
}
{{end}}
o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
{{else}}
(&o.{{$fieldInfo.Name}}[i]).decodeXDR(xr)
{{end}}
}
{{end}}
{{end}}
return xr.Error()
}`))
var maxRe = regexp.MustCompile(`\Wmax:(\d+)`)
type typeSet struct {
Type string
Encoder string
}
var xdrEncoders = map[string]typeSet{
"int16": typeSet{"uint16", "Uint16"},
"uint16": typeSet{"", "Uint16"},
"int32": typeSet{"uint32", "Uint32"},
"uint32": typeSet{"", "Uint32"},
"int64": typeSet{"uint64", "Uint64"},
"uint64": typeSet{"", "Uint64"},
"int": typeSet{"uint64", "Uint64"},
"string": typeSet{"", "String"},
"[]byte": typeSet{"", "Bytes"},
"bool": typeSet{"", "Bool"},
}
func handleStruct(t *ast.StructType) []fieldInfo {
var fs []fieldInfo
for _, sf := range t.Fields.List {
if len(sf.Names) == 0 {
// We don't handle anonymous fields
continue
}
fn := sf.Names[0].Name
var max = 0
if sf.Comment != nil {
c := sf.Comment.List[0].Text
if m := maxRe.FindStringSubmatch(c); m != nil {
max, _ = strconv.Atoi(m[1])
}
if strings.Contains(c, "noencode") {
continue
}
}
var f fieldInfo
switch ft := sf.Type.(type) {
case *ast.Ident:
tn := ft.Name
if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else {
f = fieldInfo{
Name: fn,
IsBasic: false,
FieldType: tn,
Max: max,
}
}
case *ast.ArrayType:
if ft.Len != nil {
// We don't handle arrays
continue
}
tn := ft.Elt.(*ast.Ident).Name
if enc, ok := xdrEncoders["[]"+tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
IsSlice: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max,
}
} else {
f = fieldInfo{
Name: fn,
IsBasic: false,
IsSlice: true,
FieldType: tn,
Max: max,
}
}
}
fs = append(fs, f)
}
return fs
}
func generateCode(s structInfo) {
name := s.Name
fs := s.Fields
var buf bytes.Buffer
err := encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
if err != nil {
panic(err)
}
bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
bs, err = format.Source(bs)
if err != nil {
panic(err)
}
fmt.Println(string(bs))
}
func uncamelize(s string) string {
return regexp.MustCompile("[a-z][A-Z]").ReplaceAllStringFunc(s, func(camel string) string {
return camel[:1] + " " + camel[1:]
})
}
func generateDiagram(s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Println(sn + " Structure:")
fmt.Println()
fmt.Println(" 0 1 2 3")
fmt.Println(" 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
fmt.Println(line)
for _, f := range fs {
tn := f.FieldType
sl := f.IsSlice
name := uncamelize(f.Name)
if sl {
fmt.Printf("| %s |\n", center("Number of "+name, 61))
fmt.Println(line)
}
switch tn {
case "bool":
fmt.Printf("| %s |V|\n", center(name+" (V=0 or 1)", 59))
fmt.Println(line)
case "uint16":
fmt.Printf("| %s | %s |\n", center("0x0000", 29), center(name, 29))
fmt.Println(line)
case "uint32":
fmt.Printf("| %s |\n", center(name, 61))
fmt.Println(line)
case "int64", "uint64":
fmt.Printf("| %-61s |\n", "")
fmt.Printf("+ %s +\n", center(name+" (64 bits)", 61))
fmt.Printf("| %-61s |\n", "")
fmt.Println(line)
case "string", "byte": // XXX We assume slice of byte!
fmt.Printf("| %s |\n", center("Length of "+name, 61))
fmt.Println(line)
fmt.Printf("/ %61s /\n", "")
fmt.Printf("\\ %s \\\n", center(name+" (variable length)", 61))
fmt.Printf("/ %61s /\n", "")
fmt.Println(line)
default:
if sl {
tn = "Zero or more " + tn + " Structures"
fmt.Printf("/ %s /\n", center("", 61))
fmt.Printf("\\ %s \\\n", center(tn, 61))
fmt.Printf("/ %s /\n", center("", 61))
} else {
fmt.Printf("| %s |\n", center(tn, 61))
}
fmt.Println(line)
}
}
fmt.Println()
fmt.Println()
}
func generateXdr(s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Printf("struct %s {\n", sn)
for _, f := range fs {
tn := f.FieldType
fn := f.Name
suf := ""
l := ""
if f.Max > 0 {
l = strconv.Itoa(f.Max)
}
if f.IsSlice {
suf = "<" + l + ">"
}
switch tn {
case "uint16", "uint32":
fmt.Printf("\tunsigned int %s%s;\n", fn, suf)
case "int64":
fmt.Printf("\thyper %s%s;\n", fn, suf)
case "uint64":
fmt.Printf("\tunsigned hyper %s%s;\n", fn, suf)
case "string":
fmt.Printf("\tstring %s<%s>;\n", fn, l)
case "byte":
fmt.Printf("\topaque %s<%s>;\n", fn, l)
default:
fmt.Printf("\t%s %s%s;\n", tn, fn, suf)
}
}
fmt.Println("}")
fmt.Println()
}
func center(s string, w int) string {
w -= len(s)
l := w / 2
r := l
if l+r < w {
r++
}
return strings.Repeat(" ", l) + s + strings.Repeat(" ", r)
}
func inspector(structs *[]structInfo) func(ast.Node) bool {
return func(n ast.Node) bool {
switch n := n.(type) {
case *ast.TypeSpec:
switch t := n.Type.(type) {
case *ast.StructType:
name := n.Name.Name
fs := handleStruct(t)
*structs = append(*structs, structInfo{name, fs})
}
return false
default:
return true
}
}
}
func main() {
flag.Parse()
fname := flag.Arg(0)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
if err != nil {
panic(err)
}
var structs []structInfo
i := inspector(&structs)
ast.Inspect(f, i)
headerTpl.Execute(os.Stdout, map[string]string{"Package": f.Name.Name})
for _, s := range structs {
fmt.Printf("\n/*\n\n")
generateDiagram(s)
generateXdr(s)
fmt.Printf("*/\n")
generateCode(s)
}
}

47
cmd/pidx/main.go Normal file
View File

@@ -0,0 +1,47 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/protocol"
"github.com/syndtr/goleveldb/leveldb"
)
func main() {
log.SetFlags(0)
log.SetOutput(os.Stdout)
repo := flag.String("repo", "default", "Repository ID")
node := flag.String("node", "", "Node ID (blank for global)")
flag.Parse()
db, err := leveldb.OpenFile(flag.Arg(0), nil)
if err != nil {
log.Fatal(err)
}
fs := files.NewSet(*repo, db)
if *node == "" {
log.Printf("*** Global index for repo %q", *repo)
fs.WithGlobal(func(f protocol.FileInfo) bool {
fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name))
return true
})
} else {
n, err := protocol.NodeIDFromString(*node)
if err != nil {
log.Fatal(err)
}
log.Printf("*** Have index for repo %q node %q", *repo, n)
fs.WithHave(n, func(f protocol.FileInfo) bool {
fmt.Println(f)
return true
})
}
}