vendor: Add dependencies for discosrv

vendor/github.com/cznic/fileutil/storage/cache.go

@@ -0,0 +1,322 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"container/list"
+	"io"
+	"math"
+	"os"
+	"sync"
+	"sync/atomic"
+)
+
+type cachepage struct {
+	b     [512]byte
+	dirty bool
+	lru   *list.Element
+	pi    int64
+	valid int // page content is b[:valid]
+}
+
+func (p *cachepage) wr(b []byte, off int) (wasDirty bool) {
+	copy(p.b[off:], b)
+	if n := off + len(b); n > p.valid {
+		p.valid = n
+	}
+	wasDirty = p.dirty
+	p.dirty = true
+	return
+}
+
+func (c *Cache) rd(off int64, read bool) (p *cachepage, ok bool) {
+	c.Rq++
+	pi := off >> 9
+	if p, ok = c.m[pi]; ok {
+		c.lru.MoveToBack(p.lru)
+		return
+	}
+
+	if !read {
+		return
+	}
+
+	fp := off &^ 511
+	if fp >= c.size {
+		return
+	}
+
+	rq := 512
+	if fp+512 > c.size {
+		rq = int(c.size - fp)
+	}
+	p = &cachepage{pi: pi, valid: rq}
+	p.lru = c.lru.PushBack(p)
+	if n, err := c.f.ReadAt(p.b[:p.valid], fp); n != rq {
+		panic(err)
+	}
+
+	c.Load++
+	if c.advise != nil {
+		c.advise(fp, 512, false)
+	}
+	c.m[pi], ok = p, true
+	return
+}
+
+func (c *Cache) wr(off int64) (p *cachepage) {
+	var ok bool
+	if p, ok = c.rd(off, false); ok {
+		return
+	}
+
+	pi := off >> 9
+	p = &cachepage{pi: pi}
+	p.lru = c.lru.PushBack(p)
+	c.m[pi] = p
+	return
+}
+
+// Cache provides caching support for another store Accessor.
+type Cache struct {
+	advise   func(int64, int, bool)
+	clean    chan bool
+	cleaning int32
+	close    chan bool
+	f        Accessor
+	fi       *FileInfo
+	lock     sync.Mutex
+	lru      *list.List
+	m        map[int64]*cachepage
+	maxpages int
+	size     int64
+	sync     chan bool
+	wlist    *list.List
+	write    chan bool
+	writing  int32
+	Rq       int64 // Pages requested from cache
+	Load     int64 // Pages loaded (cache miss)
+	Purge    int64 // Pages purged
+	Top      int   // "High water" pages
+}
+
+// Implementation of Accessor.
+func (c *Cache) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (c *Cache) EndUpdate() error { return nil }
+
+// NewCache creates a caching Accessor from store with total of maxcache bytes.
+// NewCache returns the new Cache, implementing Accessor or an error if any.
+//
+// The LRU mechanism is used, so the cache tries to keep often accessed pages cached.
+//
+func NewCache(store Accessor, maxcache int64, advise func(int64, int, bool)) (c *Cache, err error) {
+	var fi os.FileInfo
+	if fi, err = store.Stat(); err != nil {
+		return
+	}
+
+	x := maxcache >> 9
+	if x > math.MaxInt32/2 {
+		x = math.MaxInt32 / 2
+	}
+	c = &Cache{
+		advise:   advise,
+		clean:    make(chan bool, 1),
+		close:    make(chan bool),
+		f:        store,
+		lru:      list.New(), // front == oldest used, back == last recently used
+		m:        make(map[int64]*cachepage),
+		maxpages: int(x),
+		size:     fi.Size(),
+		sync:     make(chan bool),
+		wlist:    list.New(),
+		write:    make(chan bool, 1),
+	}
+	c.fi = NewFileInfo(fi, c)
+	go c.writer()
+	go c.cleaner(int((int64(c.maxpages) * 95) / 100)) // hysteresis
+	return
+}
+
+func (c *Cache) Accessor() Accessor {
+	return c.f
+}
+
+func (c *Cache) Close() (err error) {
+	close(c.write)
+	<-c.close
+	close(c.clean)
+	<-c.close
+	return c.f.Close()
+}
+
+func (c *Cache) Name() (s string) {
+	return c.f.Name()
+}
+
+func (c *Cache) ReadAt(b []byte, off int64) (n int, err error) {
+	po := int(off) & 0x1ff
+	bp := 0
+	rem := len(b)
+	m := 0
+	for rem != 0 {
+		c.lock.Lock() // X1+
+		p, ok := c.rd(off, true)
+		if !ok {
+			c.lock.Unlock() // X1-
+			return -1, io.EOF
+		}
+
+		rq := rem
+		if po+rq > 512 {
+			rq = 512 - po
+		}
+		if n := copy(b[bp:bp+rq], p.b[po:p.valid]); n != rq {
+			c.lock.Unlock() // X1-
+			return -1, io.EOF
+		}
+
+		m = len(c.m)
+		c.lock.Unlock() // X1-
+		po = 0
+		bp += rq
+		off += int64(rq)
+		rem -= rq
+		n += rq
+	}
+	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
+		if m > c.Top {
+			c.Top = m
+		}
+		c.clean <- true
+	}
+	return
+}
+
+func (c *Cache) Stat() (fi os.FileInfo, err error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.fi, nil
+}
+
+func (c *Cache) Sync() (err error) {
+	c.write <- false
+	<-c.sync
+	return
+}
+
+func (c *Cache) Truncate(size int64) (err error) {
+	c.Sync() //TODO improve (discard pages, the writer goroutine should also be aware, ...)
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.size = size
+	return c.f.Truncate(size)
+}
+
+func (c *Cache) WriteAt(b []byte, off int64) (n int, err error) {
+	po := int(off) & 0x1ff
+	bp := 0
+	rem := len(b)
+	m := 0
+	for rem != 0 {
+		c.lock.Lock() // X+
+		p := c.wr(off)
+		rq := rem
+		if po+rq > 512 {
+			rq = 512 - po
+		}
+		if wasDirty := p.wr(b[bp:bp+rq], po); !wasDirty {
+			c.wlist.PushBack(p)
+		}
+		m = len(c.m)
+		po = 0
+		bp += rq
+		off += int64(rq)
+		if off > c.size {
+			c.size = off
+		}
+		c.lock.Unlock() // X-
+		rem -= rq
+		n += rq
+	}
+	if atomic.CompareAndSwapInt32(&c.writing, 0, 1) {
+		c.write <- true
+	}
+	if m > c.maxpages && atomic.CompareAndSwapInt32(&c.cleaning, 0, 1) {
+		if m > c.Top {
+			c.Top = m
+		}
+		c.clean <- true
+	}
+	return
+}
+
+func (c *Cache) writer() {
+	for ok := true; ok; {
+		var wr bool
+		var off int64
+		wr, ok = <-c.write
+		for {
+			c.lock.Lock() // X1+
+			item := c.wlist.Front()
+			if item == nil {
+				c.lock.Unlock() // X1-
+				break
+			}
+
+			p := item.Value.(*cachepage)
+			off = p.pi << 9
+			if n, err := c.f.WriteAt(p.b[:p.valid], off); n != p.valid {
+				c.lock.Unlock()                    // X1-
+				panic("TODO Cache.writer errchan") //TODO +errchan
+				panic(err)
+			}
+
+			p.dirty = false
+			c.wlist.Remove(item)
+			if c.advise != nil {
+				c.advise(off, 512, true)
+			}
+			c.lock.Unlock() // X1-
+		}
+		switch {
+		case wr:
+			atomic.AddInt32(&c.writing, -1)
+		case ok:
+			c.sync <- true
+		}
+	}
+	c.close <- true
+}
+
+func (c *Cache) cleaner(limit int) {
+	for _ = range c.clean {
+		var item *list.Element
+		for {
+			c.lock.Lock() // X1+
+			if len(c.m) < limit {
+				c.lock.Unlock() // X1-
+				break
+			}
+
+			if item == nil {
+				item = c.lru.Front()
+			}
+			if p := item.Value.(*cachepage); !p.dirty {
+				delete(c.m, p.pi)
+				c.lru.Remove(item)
+				c.Purge++
+			}
+			item = item.Next()
+			c.lock.Unlock() // X1-
+		}
+		atomic.AddInt32(&c.cleaning, -1)
+	}
+	c.close <- true
+}

vendor/github.com/cznic/fileutil/storage/file.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"os"
+)
+
+// FileAccessor is the concrete type returned by NewFile and OpenFile.
+type FileAccessor struct {
+	*os.File
+}
+
+// Implementation of Accessor.
+func (f *FileAccessor) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (f *FileAccessor) EndUpdate() error { return nil }
+
+// NewFile returns an Accessor backed by an os.File named name, It opens the
+// named file with specified flag (os.O_RDWR etc.) and perm, (0666 etc.) if
+// applicable.  If successful, methods on the returned Accessor can be used for
+// I/O.  It returns the Accessor and an Error, if any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func NewFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
+	var f FileAccessor
+	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
+		store = &f
+	}
+	return
+}
+
+// OpenFile returns an Accessor backed by an existing os.File named name, It
+// opens the named file with specified flag (os.O_RDWR etc.) and perm, (0666
+// etc.) if applicable.  If successful, methods on the returned Accessor can be
+// used for I/O.  It returns the Accessor and an Error, if any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func OpenFile(name string, flag int, perm os.FileMode) (store Accessor, err error) {
+	var f FileAccessor
+	if f.File, err = os.OpenFile(name, flag, perm); err == nil {
+		store = &f
+	}
+	return
+}

vendor/github.com/cznic/fileutil/storage/mem.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"os"
+)
+
+//TODO -> exported type w/ exported fields
+type memaccessor struct {
+	f  *os.File
+	fi *FileInfo
+	b  []byte
+}
+
+// Implementation of Accessor.
+func (m *memaccessor) BeginUpdate() error { return nil }
+
+// Implementation of Accessor.
+func (f *memaccessor) EndUpdate() error { return nil }
+
+// NewMem returns a new Accessor backed by an os.File.  The returned Accessor
+// keeps all of the store content in memory.  The memory and file images are
+// synced only by Sync and Close.  Recomended for small amounts of data only
+// and content which may be lost on process kill/crash.  NewMem return the
+// Accessor or an error of any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func NewMem(f *os.File) (store Accessor, err error) {
+	a := &memaccessor{f: f}
+	if err = f.Truncate(0); err != nil {
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = a.f.Stat(); err != nil {
+		return
+	}
+
+	a.fi = NewFileInfo(fi, a)
+	store = a
+	return
+}
+
+// OpenMem return a new Accessor backed by an os.File.  The store content is
+// loaded from f.  The returned Accessor keeps all of the store content in
+// memory.  The memory and file images are synced only Sync and Close.
+// Recomended for small amounts of data only and content which may be lost on
+// process kill/crash.  OpenMem return the Accessor or an error of any.
+//
+// NOTE: The returned Accessor implements BeginUpdate and EndUpdate as a no op.
+func OpenMem(f *os.File) (store Accessor, err error) {
+	a := &memaccessor{f: f}
+	if a.b, err = ioutil.ReadAll(a.f); err != nil {
+		a.f.Close()
+		return
+	}
+
+	var fi os.FileInfo
+	if fi, err = a.f.Stat(); err != nil {
+		a.f.Close()
+		return
+	}
+
+	a.fi = NewFileInfo(fi, a)
+	store = a
+	return
+}
+
+// Close implements Accessor. Specifically it synchronizes the memory and file images.
+func (a *memaccessor) Close() (err error) {
+	defer func() {
+		a.b = nil
+		if a.f != nil {
+			if e := a.f.Close(); e != nil && err == nil {
+				err = e
+			}
+		}
+		a.f = nil
+	}()
+
+	return a.Sync()
+}
+
+func (a *memaccessor) Name() string {
+	return a.f.Name()
+}
+
+func (a *memaccessor) ReadAt(b []byte, off int64) (n int, err error) {
+	if off < 0 || off > math.MaxInt32 {
+		return -1, fmt.Errorf("ReadAt: illegal offset %#x", off)
+	}
+
+	rq, fp := len(b), int(off)
+	if fp+rq > len(a.b) {
+		return -1, fmt.Errorf("ReadAt: illegal rq %#x @ offset %#x, len %#x", rq, fp, len(a.b))
+	}
+
+	copy(b, a.b[fp:])
+	return
+}
+
+func (a *memaccessor) Stat() (fi os.FileInfo, err error) {
+	i := a.fi
+	i.FSize = int64(len(a.b))
+	fi = i
+	return
+}
+
+// Sync implements Accessor. Specifically it synchronizes the memory and file images.
+func (a *memaccessor) Sync() (err error) {
+	var n int
+	if n, err = a.f.WriteAt(a.b, 0); n != len(a.b) {
+		return
+	}
+
+	return a.f.Truncate(int64(len(a.b)))
+}
+
+func (a *memaccessor) Truncate(size int64) (err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			err = e.(error)
+		}
+	}()
+
+	if size > math.MaxInt32 {
+		panic(errors.New("truncate: illegal size"))
+	}
+
+	a.b = a.b[:int(size)]
+	return
+}
+
+func (a *memaccessor) WriteAt(b []byte, off int64) (n int, err error) {
+	if off < 0 || off > math.MaxInt32 {
+		return -1, errors.New("WriteAt: illegal offset")
+	}
+
+	rq, fp, size := len(b), int(off), len(a.b)
+	if need := rq + fp; need > size {
+		if need <= cap(a.b) {
+			a.b = a.b[:need]
+		} else {
+			nb := make([]byte, need, 2*need)
+			copy(nb, a.b)
+			a.b = nb
+		}
+	}
+
+	copy(a.b[int(off):], b)
+	return
+}

vendor/github.com/cznic/fileutil/storage/probe.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+import "sync/atomic"
+
+// Probe collects usage statistics of the embeded Accessor.
+// Probe itself IS an Accessor.
+type Probe struct {
+	Accessor
+	Chain     *Probe
+	OpsRd     int64
+	OpsWr     int64
+	BytesRd   int64
+	BytesWr   int64
+	SectorsRd int64 // Assuming 512 byte sector size
+	SectorsWr int64
+}
+
+// NewProbe returns a newly created probe which embedes the src Accessor.
+// The retuned *Probe satisfies Accessor. if chain != nil then Reset()
+// is cascaded down the chained Probes.
+func NewProbe(src Accessor, chain *Probe) *Probe {
+	return &Probe{Accessor: src, Chain: chain}
+}
+
+func reset(n *int64) {
+	atomic.AddInt64(n, -atomic.AddInt64(n, 0))
+}
+
+// Reset zeroes the collected statistics of p.
+func (p *Probe) Reset() {
+	if p.Chain != nil {
+		p.Chain.Reset()
+	}
+	reset(&p.OpsRd)
+	reset(&p.OpsWr)
+	reset(&p.BytesRd)
+	reset(&p.BytesWr)
+	reset(&p.SectorsRd)
+	reset(&p.SectorsWr)
+}
+
+func (p *Probe) ReadAt(b []byte, off int64) (n int, err error) {
+	n, err = p.Accessor.ReadAt(b, off)
+	atomic.AddInt64(&p.OpsRd, 1)
+	atomic.AddInt64(&p.BytesRd, int64(n))
+	if n <= 0 {
+		return
+	}
+
+	sectorFirst := off >> 9
+	sectorLast := (off + int64(n) - 1) >> 9
+	atomic.AddInt64(&p.SectorsRd, sectorLast-sectorFirst+1)
+	return
+}
+
+func (p *Probe) WriteAt(b []byte, off int64) (n int, err error) {
+	n, err = p.Accessor.WriteAt(b, off)
+	atomic.AddInt64(&p.OpsWr, 1)
+	atomic.AddInt64(&p.BytesWr, int64(n))
+	if n <= 0 {
+		return
+	}
+
+	sectorFirst := off >> 9
+	sectorLast := (off + int64(n) - 1) >> 9
+	atomic.AddInt64(&p.SectorsWr, sectorLast-sectorFirst+1)
+	return
+}

vendor/github.com/cznic/fileutil/storage/storage.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+// WIP: Package storage defines and implements storage providers and store accessors.
+package storage
+
+import (
+	"os"
+	"sync"
+	"time"
+)
+
+// FileInfo is a type implementing os.FileInfo which has setable fields, like
+// the older os.FileInfo used to have. It is used wehere e.g. the Size is
+// needed to be faked (encapsulated/memory only file, file cache, etc.).
+type FileInfo struct {
+	FName    string      // base name of the file
+	FSize    int64       // length in bytes
+	FMode    os.FileMode // file mode bits
+	FModTime time.Time   // modification time
+	FIsDir   bool        // abbreviation for Mode().IsDir()
+	sys      interface{} // underlying data source (can be nil)
+}
+
+// NewFileInfo creates FileInfo from os.FileInfo fi.
+func NewFileInfo(fi os.FileInfo, sys interface{}) *FileInfo {
+	return &FileInfo{fi.Name(), fi.Size(), fi.Mode(), fi.ModTime(), fi.IsDir(), sys}
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Name() string {
+	return fi.FName
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Size() int64 {
+	return fi.FSize
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) Mode() os.FileMode {
+	return fi.FMode
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) ModTime() time.Time {
+	return fi.FModTime
+}
+
+// Implementation of os.FileInfo
+func (fi *FileInfo) IsDir() bool {
+	return fi.FIsDir
+}
+
+func (fi *FileInfo) Sys() interface{} {
+	return fi.sys
+}
+
+// Accessor provides I/O methods to access a store.
+type Accessor interface {
+
+	// Close closes the store, rendering it unusable for I/O. It returns an
+	// error, if any.
+	Close() error
+
+	// Name returns the name of the file as presented to Open.
+	Name() string
+
+	// ReadAt reads len(b) bytes from the store starting at byte offset off.
+	// It returns the number of bytes read and the error, if any.
+	// EOF is signaled by a zero count with err set to os.EOF.
+	// ReadAt always returns a non-nil Error when n != len(b).
+	ReadAt(b []byte, off int64) (n int, err error)
+
+	// Stat returns the FileInfo structure describing the store. It returns
+	// the os.FileInfo and an error, if any.
+	Stat() (fi os.FileInfo, err error)
+
+	// Sync commits the current contents of the store to stable storage.
+	// Typically, this means flushing the file system's in-memory copy of
+	// recently written data to disk.
+	Sync() (err error)
+
+	// Truncate changes the size of the store. It does not change the I/O
+	// offset.
+	Truncate(size int64) error
+
+	// WriteAt writes len(b) bytes to the store starting at byte offset off.
+	// It returns the number of bytes written and an error, if any.
+	// WriteAt returns a non-nil Error when n != len(b).
+	WriteAt(b []byte, off int64) (n int, err error)
+
+	// Before every [structural] change of a store the BeginUpdate is to be
+	// called and paired with EndUpdate after the change makes the store's
+	// state consistent again. Invocations of BeginUpdate may nest. On
+	// invoking the last non nested EndUpdate an implicit "commit" should
+	// be performed by the store/provider. The concrete mechanism is
+	// unspecified. It could be for example a write-ahead log.  Stores may
+	// implement BeginUpdate and EndUpdate as a (documented) no op.
+	BeginUpdate() error
+	EndUpdate() error
+}
+
+// Mutate is a helper/wrapper for executing f in between a.BeginUpdate and
+// a.EndUpdate.  Any parameters and/or return values except an error should be
+// captured by a function literal passed as f. The returned err is either nil
+// or the first non nil error returned from the sequence of execution:
+// BeginUpdate, [f,] EndUpdate. The pair BeginUpdate/EndUpdate *is* invoked
+// always regardles of any possible errors produced.  Mutate doesn't handle
+// panic, it should be used only with a function [literal] which doesn't panic.
+// Otherwise the pairing of BeginUpdate/EndUpdate is not guaranteed.
+//
+// NOTE: If BeginUpdate, which is invoked before f, returns a non-nil error,
+// then f is not invoked at all (but EndUpdate still is).
+func Mutate(a Accessor, f func() error) (err error) {
+	defer func() {
+		if e := a.EndUpdate(); e != nil && err == nil {
+			err = e
+		}
+	}()
+
+	if err = a.BeginUpdate(); err != nil {
+		return
+	}
+
+	return f()
+}
+
+// LockedMutate wraps Mutate in yet another layer consisting of a
+// l.Lock/l.Unlock pair. All other limitations apply as in Mutate, e.g. no
+// panics are allowed to happen - otherwise no guarantees can be made about
+// Unlock matching the Lock.
+func LockedMutate(a Accessor, l sync.Locker, f func() error) (err error) {
+	l.Lock()
+	defer l.Unlock()
+
+	return Mutate(a, f)
+}

vendor/github.com/cznic/fileutil/storage/test_deps.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// blame: jnml, labs.nic.cz
+
+package storage
+
+// Pull test dependencies too.
+// Enables easy 'go test X' after 'go get X'
+import (
+// nothing yet
+)