cmd/stdiscosrv: New discovery server (fixes #4618)

This is a new revision of the discovery server. Relevant changes and
non-changes:

- Protocol towards clients is unchanged.

- Recommended large scale design is still to be deployed nehind nginx (I
  tested, and it's still a lot faster at terminating TLS).

- Database backend is leveldb again, only. It scales enough, is easy to
  setup, and we don't need any backend to take care of.

- Server supports replication. This is a simple TCP channel - protect it
  with a firewall when deploying over the internet. (We deploy this within
  the same datacenter, and with firewall.) Any incoming client announces
  are sent over the replication channel(s) to other peer discosrvs.
  Incoming replication changes are applied to the database as if they came
  from clients, but without the TLS/certificate overhead.

- Metrics are exposed using the prometheus library, when enabled.

- The database values and replication protocol is protobuf, because JSON
  was quite CPU intensive when I tried that and benchmarked it.

- The "Retry-After" value for failed lookups gets slowly increased from
  a default of 120 seconds, by 5 seconds for each failed lookup,
  independently by each discosrv. This lowers the query load over time for
  clients that are never seen. The Retry-After maxes out at 3600 after a
  couple of weeks of this increase. The number of failed lookups is
  stored in the database, now and then (avoiding making each lookup a
  database put).

All in all this means clients can be pointed towards a cluster using
just multiple A / AAAA records to gain both load sharing and redundancy
(if one is down, clients will talk to the remaining ones).

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4648

vendor/github.com/cheggaaa/pb/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2012-2015, Sergey Cherepanov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/cheggaaa/pb/format.go

@@ -0,0 +1,118 @@
+package pb
+
+import (
+	"fmt"
+	"time"
+)
+
+type Units int
+
+const (
+	// U_NO are default units, they represent a simple value and are not formatted at all.
+	U_NO Units = iota
+	// U_BYTES units are formatted in a human readable way (B, KiB, MiB, ...)
+	U_BYTES
+	// U_BYTES_DEC units are like U_BYTES, but base 10 (B, KB, MB, ...)
+	U_BYTES_DEC
+	// U_DURATION units are formatted in a human readable way (3h14m15s)
+	U_DURATION
+)
+
+const (
+	KiB = 1024
+	MiB = 1048576
+	GiB = 1073741824
+	TiB = 1099511627776
+
+	KB = 1e3
+	MB = 1e6
+	GB = 1e9
+	TB = 1e12
+)
+
+func Format(i int64) *formatter {
+	return &formatter{n: i}
+}
+
+type formatter struct {
+	n      int64
+	unit   Units
+	width  int
+	perSec bool
+}
+
+func (f *formatter) To(unit Units) *formatter {
+	f.unit = unit
+	return f
+}
+
+func (f *formatter) Width(width int) *formatter {
+	f.width = width
+	return f
+}
+
+func (f *formatter) PerSec() *formatter {
+	f.perSec = true
+	return f
+}
+
+func (f *formatter) String() (out string) {
+	switch f.unit {
+	case U_BYTES:
+		out = formatBytes(f.n)
+	case U_BYTES_DEC:
+		out = formatBytesDec(f.n)
+	case U_DURATION:
+		out = formatDuration(f.n)
+	default:
+		out = fmt.Sprintf(fmt.Sprintf("%%%dd", f.width), f.n)
+	}
+	if f.perSec {
+		out += "/s"
+	}
+	return
+}
+
+// Convert bytes to human readable string. Like 2 MiB, 64.2 KiB, 52 B
+func formatBytes(i int64) (result string) {
+	switch {
+	case i >= TiB:
+		result = fmt.Sprintf("%.02f TiB", float64(i)/TiB)
+	case i >= GiB:
+		result = fmt.Sprintf("%.02f GiB", float64(i)/GiB)
+	case i >= MiB:
+		result = fmt.Sprintf("%.02f MiB", float64(i)/MiB)
+	case i >= KiB:
+		result = fmt.Sprintf("%.02f KiB", float64(i)/KiB)
+	default:
+		result = fmt.Sprintf("%d B", i)
+	}
+	return
+}
+
+// Convert bytes to base-10 human readable string. Like 2 MB, 64.2 KB, 52 B
+func formatBytesDec(i int64) (result string) {
+	switch {
+	case i >= TB:
+		result = fmt.Sprintf("%.02f TB", float64(i)/TB)
+	case i >= GB:
+		result = fmt.Sprintf("%.02f GB", float64(i)/GB)
+	case i >= MB:
+		result = fmt.Sprintf("%.02f MB", float64(i)/MB)
+	case i >= KB:
+		result = fmt.Sprintf("%.02f KB", float64(i)/KB)
+	default:
+		result = fmt.Sprintf("%d B", i)
+	}
+	return
+}
+
+func formatDuration(n int64) (result string) {
+	d := time.Duration(n)
+	if d > time.Hour*24 {
+		result = fmt.Sprintf("%dd", d/24/time.Hour)
+		d -= (d / time.Hour / 24) * (time.Hour * 24)
+	}
+	result = fmt.Sprintf("%s%v", result, d)
+	return
+}

vendor/github.com/cheggaaa/pb/pb.go

@@ -0,0 +1,469 @@
+// Simple console progress bars
+package pb
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unicode/utf8"
+)
+
+// Current version
+const Version = "1.0.19"
+
+const (
+	// Default refresh rate - 200ms
+	DEFAULT_REFRESH_RATE = time.Millisecond * 200
+	FORMAT               = "[=>-]"
+)
+
+// DEPRECATED
+// variables for backward compatibility, from now do not work
+// use pb.Format and pb.SetRefreshRate
+var (
+	DefaultRefreshRate                         = DEFAULT_REFRESH_RATE
+	BarStart, BarEnd, Empty, Current, CurrentN string
+)
+
+// Create new progress bar object
+func New(total int) *ProgressBar {
+	return New64(int64(total))
+}
+
+// Create new progress bar object using int64 as total
+func New64(total int64) *ProgressBar {
+	pb := &ProgressBar{
+		Total:         total,
+		RefreshRate:   DEFAULT_REFRESH_RATE,
+		ShowPercent:   true,
+		ShowCounters:  true,
+		ShowBar:       true,
+		ShowTimeLeft:  true,
+		ShowFinalTime: true,
+		Units:         U_NO,
+		ManualUpdate:  false,
+		finish:        make(chan struct{}),
+	}
+	return pb.Format(FORMAT)
+}
+
+// Create new object and start
+func StartNew(total int) *ProgressBar {
+	return New(total).Start()
+}
+
+// Callback for custom output
+// For example:
+// bar.Callback = func(s string) {
+//     mySuperPrint(s)
+// }
+//
+type Callback func(out string)
+
+type ProgressBar struct {
+	current  int64 // current must be first member of struct (https://code.google.com/p/go/issues/detail?id=5278)
+	previous int64
+
+	Total                            int64
+	RefreshRate                      time.Duration
+	ShowPercent, ShowCounters        bool
+	ShowSpeed, ShowTimeLeft, ShowBar bool
+	ShowFinalTime                    bool
+	Output                           io.Writer
+	Callback                         Callback
+	NotPrint                         bool
+	Units                            Units
+	Width                            int
+	ForceWidth                       bool
+	ManualUpdate                     bool
+	AutoStat                         bool
+
+	// Default width for the time box.
+	UnitsWidth   int
+	TimeBoxWidth int
+
+	finishOnce sync.Once //Guards isFinish
+	finish     chan struct{}
+	isFinish   bool
+
+	startTime  time.Time
+	startValue int64
+
+	changeTime time.Time
+
+	prefix, postfix string
+
+	mu        sync.Mutex
+	lastPrint string
+
+	BarStart string
+	BarEnd   string
+	Empty    string
+	Current  string
+	CurrentN string
+
+	AlwaysUpdate bool
+}
+
+// Start print
+func (pb *ProgressBar) Start() *ProgressBar {
+	pb.startTime = time.Now()
+	pb.startValue = atomic.LoadInt64(&pb.current)
+	if pb.Total == 0 {
+		pb.ShowTimeLeft = false
+		pb.ShowPercent = false
+		pb.AutoStat = false
+	}
+	if !pb.ManualUpdate {
+		pb.Update() // Initial printing of the bar before running the bar refresher.
+		go pb.refresher()
+	}
+	return pb
+}
+
+// Increment current value
+func (pb *ProgressBar) Increment() int {
+	return pb.Add(1)
+}
+
+// Get current value
+func (pb *ProgressBar) Get() int64 {
+	c := atomic.LoadInt64(&pb.current)
+	return c
+}
+
+// Set current value
+func (pb *ProgressBar) Set(current int) *ProgressBar {
+	return pb.Set64(int64(current))
+}
+
+// Set64 sets the current value as int64
+func (pb *ProgressBar) Set64(current int64) *ProgressBar {
+	atomic.StoreInt64(&pb.current, current)
+	return pb
+}
+
+// Add to current value
+func (pb *ProgressBar) Add(add int) int {
+	return int(pb.Add64(int64(add)))
+}
+
+func (pb *ProgressBar) Add64(add int64) int64 {
+	return atomic.AddInt64(&pb.current, add)
+}
+
+// Set prefix string
+func (pb *ProgressBar) Prefix(prefix string) *ProgressBar {
+	pb.prefix = prefix
+	return pb
+}
+
+// Set postfix string
+func (pb *ProgressBar) Postfix(postfix string) *ProgressBar {
+	pb.postfix = postfix
+	return pb
+}
+
+// Set custom format for bar
+// Example: bar.Format("[=>_]")
+// Example: bar.Format("[\x00=\x00>\x00-\x00]") // \x00 is the delimiter
+func (pb *ProgressBar) Format(format string) *ProgressBar {
+	var formatEntries []string
+	if utf8.RuneCountInString(format) == 5 {
+		formatEntries = strings.Split(format, "")
+	} else {
+		formatEntries = strings.Split(format, "\x00")
+	}
+	if len(formatEntries) == 5 {
+		pb.BarStart = formatEntries[0]
+		pb.BarEnd = formatEntries[4]
+		pb.Empty = formatEntries[3]
+		pb.Current = formatEntries[1]
+		pb.CurrentN = formatEntries[2]
+	}
+	return pb
+}
+
+// Set bar refresh rate
+func (pb *ProgressBar) SetRefreshRate(rate time.Duration) *ProgressBar {
+	pb.RefreshRate = rate
+	return pb
+}
+
+// Set units
+// bar.SetUnits(U_NO) - by default
+// bar.SetUnits(U_BYTES) - for Mb, Kb, etc
+func (pb *ProgressBar) SetUnits(units Units) *ProgressBar {
+	pb.Units = units
+	return pb
+}
+
+// Set max width, if width is bigger than terminal width, will be ignored
+func (pb *ProgressBar) SetMaxWidth(width int) *ProgressBar {
+	pb.Width = width
+	pb.ForceWidth = false
+	return pb
+}
+
+// Set bar width
+func (pb *ProgressBar) SetWidth(width int) *ProgressBar {
+	pb.Width = width
+	pb.ForceWidth = true
+	return pb
+}
+
+// End print
+func (pb *ProgressBar) Finish() {
+	//Protect multiple calls
+	pb.finishOnce.Do(func() {
+		close(pb.finish)
+		pb.write(atomic.LoadInt64(&pb.current))
+		pb.mu.Lock()
+		defer pb.mu.Unlock()
+		switch {
+		case pb.Output != nil:
+			fmt.Fprintln(pb.Output)
+		case !pb.NotPrint:
+			fmt.Println()
+		}
+		pb.isFinish = true
+	})
+}
+
+// IsFinished return boolean
+func (pb *ProgressBar) IsFinished() bool {
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	return pb.isFinish
+}
+
+// End print and write string 'str'
+func (pb *ProgressBar) FinishPrint(str string) {
+	pb.Finish()
+	if pb.Output != nil {
+		fmt.Fprintln(pb.Output, str)
+	} else {
+		fmt.Println(str)
+	}
+}
+
+// implement io.Writer
+func (pb *ProgressBar) Write(p []byte) (n int, err error) {
+	n = len(p)
+	pb.Add(n)
+	return
+}
+
+// implement io.Reader
+func (pb *ProgressBar) Read(p []byte) (n int, err error) {
+	n = len(p)
+	pb.Add(n)
+	return
+}
+
+// Create new proxy reader over bar
+// Takes io.Reader or io.ReadCloser
+func (pb *ProgressBar) NewProxyReader(r io.Reader) *Reader {
+	return &Reader{r, pb}
+}
+
+func (pb *ProgressBar) write(current int64) {
+	width := pb.GetWidth()
+
+	var percentBox, countersBox, timeLeftBox, speedBox, barBox, end, out string
+
+	// percents
+	if pb.ShowPercent {
+		var percent float64
+		if pb.Total > 0 {
+			percent = float64(current) / (float64(pb.Total) / float64(100))
+		} else {
+			percent = float64(current) / float64(100)
+		}
+		percentBox = fmt.Sprintf(" %6.02f%%", percent)
+	}
+
+	// counters
+	if pb.ShowCounters {
+		current := Format(current).To(pb.Units).Width(pb.UnitsWidth)
+		if pb.Total > 0 {
+			total := Format(pb.Total).To(pb.Units).Width(pb.UnitsWidth)
+			countersBox = fmt.Sprintf(" %s / %s ", current, total)
+		} else {
+			countersBox = fmt.Sprintf(" %s / ? ", current)
+		}
+	}
+
+	// time left
+	pb.mu.Lock()
+	currentFromStart := current - pb.startValue
+	fromStart := time.Now().Sub(pb.startTime)
+	lastChangeTime := pb.changeTime
+	fromChange := lastChangeTime.Sub(pb.startTime)
+	pb.mu.Unlock()
+	select {
+	case <-pb.finish:
+		if pb.ShowFinalTime {
+			var left time.Duration
+			left = (fromStart / time.Second) * time.Second
+			timeLeftBox = fmt.Sprintf(" %s", left.String())
+		}
+	default:
+		if pb.ShowTimeLeft && currentFromStart > 0 {
+			perEntry := fromChange / time.Duration(currentFromStart)
+			var left time.Duration
+			if pb.Total > 0 {
+				left = time.Duration(pb.Total-currentFromStart) * perEntry
+				left -= time.Since(lastChangeTime)
+				left = (left / time.Second) * time.Second
+			} else {
+				left = time.Duration(currentFromStart) * perEntry
+				left = (left / time.Second) * time.Second
+			}
+			if left > 0 {
+				timeLeft := Format(int64(left)).To(U_DURATION).String()
+				timeLeftBox = fmt.Sprintf(" %s", timeLeft)
+			}
+		}
+	}
+
+	if len(timeLeftBox) < pb.TimeBoxWidth {
+		timeLeftBox = fmt.Sprintf("%s%s", strings.Repeat(" ", pb.TimeBoxWidth-len(timeLeftBox)), timeLeftBox)
+	}
+
+	// speed
+	if pb.ShowSpeed && currentFromStart > 0 {
+		fromStart := time.Now().Sub(pb.startTime)
+		speed := float64(currentFromStart) / (float64(fromStart) / float64(time.Second))
+		speedBox = " " + Format(int64(speed)).To(pb.Units).Width(pb.UnitsWidth).PerSec().String()
+	}
+
+	barWidth := escapeAwareRuneCountInString(countersBox + pb.BarStart + pb.BarEnd + percentBox + timeLeftBox + speedBox + pb.prefix + pb.postfix)
+	// bar
+	if pb.ShowBar {
+		size := width - barWidth
+		if size > 0 {
+			if pb.Total > 0 {
+				curSize := int(math.Ceil((float64(current) / float64(pb.Total)) * float64(size)))
+				emptySize := size - curSize
+				barBox = pb.BarStart
+				if emptySize < 0 {
+					emptySize = 0
+				}
+				if curSize > size {
+					curSize = size
+				}
+
+				cursorLen := escapeAwareRuneCountInString(pb.Current)
+				if emptySize <= 0 {
+					barBox += strings.Repeat(pb.Current, curSize/cursorLen)
+				} else if curSize > 0 {
+					cursorEndLen := escapeAwareRuneCountInString(pb.CurrentN)
+					cursorRepetitions := (curSize - cursorEndLen) / cursorLen
+					barBox += strings.Repeat(pb.Current, cursorRepetitions)
+					barBox += pb.CurrentN
+				}
+
+				emptyLen := escapeAwareRuneCountInString(pb.Empty)
+				barBox += strings.Repeat(pb.Empty, emptySize/emptyLen)
+				barBox += pb.BarEnd
+			} else {
+				pos := size - int(current)%int(size)
+				barBox = pb.BarStart
+				if pos-1 > 0 {
+					barBox += strings.Repeat(pb.Empty, pos-1)
+				}
+				barBox += pb.Current
+				if size-pos-1 > 0 {
+					barBox += strings.Repeat(pb.Empty, size-pos-1)
+				}
+				barBox += pb.BarEnd
+			}
+		}
+	}
+
+	// check len
+	out = pb.prefix + countersBox + barBox + percentBox + speedBox + timeLeftBox + pb.postfix
+	if cl := escapeAwareRuneCountInString(out); cl < width {
+		end = strings.Repeat(" ", width-cl)
+	}
+
+	// and print!
+	pb.mu.Lock()
+	pb.lastPrint = out + end
+	isFinish := pb.isFinish
+	pb.mu.Unlock()
+	switch {
+	case isFinish:
+		return
+	case pb.Output != nil:
+		fmt.Fprint(pb.Output, "\r"+out+end)
+	case pb.Callback != nil:
+		pb.Callback(out + end)
+	case !pb.NotPrint:
+		fmt.Print("\r" + out + end)
+	}
+}
+
+// GetTerminalWidth - returns terminal width for all platforms.
+func GetTerminalWidth() (int, error) {
+	return terminalWidth()
+}
+
+func (pb *ProgressBar) GetWidth() int {
+	if pb.ForceWidth {
+		return pb.Width
+	}
+
+	width := pb.Width
+	termWidth, _ := terminalWidth()
+	if width == 0 || termWidth <= width {
+		width = termWidth
+	}
+
+	return width
+}
+
+// Write the current state of the progressbar
+func (pb *ProgressBar) Update() {
+	c := atomic.LoadInt64(&pb.current)
+	p := atomic.LoadInt64(&pb.previous)
+	if p != c {
+		pb.mu.Lock()
+		pb.changeTime = time.Now()
+		pb.mu.Unlock()
+		atomic.StoreInt64(&pb.previous, c)
+	}
+	pb.write(c)
+	if pb.AutoStat {
+		if c == 0 {
+			pb.startTime = time.Now()
+			pb.startValue = 0
+		} else if c >= pb.Total && pb.isFinish != true {
+			pb.Finish()
+		}
+	}
+}
+
+// String return the last bar print
+func (pb *ProgressBar) String() string {
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	return pb.lastPrint
+}
+
+// Internal loop for refreshing the progressbar
+func (pb *ProgressBar) refresher() {
+	for {
+		select {
+		case <-pb.finish:
+			return
+		case <-time.After(pb.RefreshRate):
+			pb.Update()
+		}
+	}
+}

vendor/github.com/cheggaaa/pb/pb_appengine.go

@@ -0,0 +1,11 @@
+// +build appengine
+
+package pb
+
+import "errors"
+
+// terminalWidth returns width of the terminal, which is not supported
+// and should always failed on appengine classic which is a sandboxed PaaS.
+func terminalWidth() (int, error) {
+	return 0, errors.New("Not supported")
+}

vendor/github.com/cheggaaa/pb/pb_win.go

@@ -0,0 +1,141 @@
+// +build windows
+
+package pb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+	"syscall"
+	"unsafe"
+)
+
+var tty = os.Stdin
+
+var (
+	kernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+	// GetConsoleScreenBufferInfo retrieves information about the
+	// specified console screen buffer.
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx
+	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
+
+	// GetConsoleMode retrieves the current input mode of a console's
+	// input buffer or the current output mode of a console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms683167(v=vs.85).aspx
+	getConsoleMode = kernel32.NewProc("GetConsoleMode")
+
+	// SetConsoleMode sets the input mode of a console's input buffer
+	// or the output mode of a console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx
+	setConsoleMode = kernel32.NewProc("SetConsoleMode")
+
+	// SetConsoleCursorPosition sets the cursor position in the
+	// specified console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms686025(v=vs.85).aspx
+	setConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition")
+)
+
+type (
+	// Defines the coordinates of the upper left and lower right corners
+	// of a rectangle.
+	// See
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx
+	smallRect struct {
+		Left, Top, Right, Bottom int16
+	}
+
+	// Defines the coordinates of a character cell in a console screen
+	// buffer. The origin of the coordinate system (0,0) is at the top, left cell
+	// of the buffer.
+	// See
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx
+	coordinates struct {
+		X, Y int16
+	}
+
+	word int16
+
+	// Contains information about a console screen buffer.
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx
+	consoleScreenBufferInfo struct {
+		dwSize              coordinates
+		dwCursorPosition    coordinates
+		wAttributes         word
+		srWindow            smallRect
+		dwMaximumWindowSize coordinates
+	}
+)
+
+// terminalWidth returns width of the terminal.
+func terminalWidth() (width int, err error) {
+	var info consoleScreenBufferInfo
+	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
+	if e != 0 {
+		return 0, error(e)
+	}
+	return int(info.dwSize.X) - 1, nil
+}
+
+func getCursorPos() (pos coordinates, err error) {
+	var info consoleScreenBufferInfo
+	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
+	if e != 0 {
+		return info.dwCursorPosition, error(e)
+	}
+	return info.dwCursorPosition, nil
+}
+
+func setCursorPos(pos coordinates) error {
+	_, _, e := syscall.Syscall(setConsoleCursorPosition.Addr(), 2, uintptr(syscall.Stdout), uintptr(uint32(uint16(pos.Y))<<16|uint32(uint16(pos.X))), 0)
+	if e != 0 {
+		return error(e)
+	}
+	return nil
+}
+
+var ErrPoolWasStarted = errors.New("Bar pool was started")
+
+var echoLocked bool
+var echoLockMutex sync.Mutex
+
+var oldState word
+
+func lockEcho() (quit chan int, err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if echoLocked {
+		err = ErrPoolWasStarted
+		return
+	}
+	echoLocked = true
+
+	if _, _, e := syscall.Syscall(getConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&oldState)), 0); e != 0 {
+		err = fmt.Errorf("Can't get terminal settings: %v", e)
+		return
+	}
+
+	newState := oldState
+	const ENABLE_ECHO_INPUT = 0x0004
+	const ENABLE_LINE_INPUT = 0x0002
+	newState = newState & (^(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT))
+	if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(newState), 0); e != 0 {
+		err = fmt.Errorf("Can't set terminal settings: %v", e)
+		return
+	}
+	return
+}
+
+func unlockEcho() (err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if !echoLocked {
+		return
+	}
+	echoLocked = false
+	if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(oldState), 0); e != 0 {
+		err = fmt.Errorf("Can't set terminal settings")
+	}
+	return
+}

vendor/github.com/cheggaaa/pb/pb_x.go

@@ -0,0 +1,108 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly
+// +build !appengine
+
+package pb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+var ErrPoolWasStarted = errors.New("Bar pool was started")
+
+var (
+	echoLockMutex    sync.Mutex
+	origTermStatePtr *unix.Termios
+	tty              *os.File
+)
+
+func init() {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+
+	var err error
+	tty, err = os.Open("/dev/tty")
+	if err != nil {
+		tty = os.Stdin
+	}
+}
+
+// terminalWidth returns width of the terminal.
+func terminalWidth() (int, error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+
+	fd := int(tty.Fd())
+
+	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
+	if err != nil {
+		return 0, err
+	}
+
+	return int(ws.Col), nil
+}
+
+func lockEcho() (quit chan int, err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if origTermStatePtr != nil {
+		return quit, ErrPoolWasStarted
+	}
+
+	fd := int(tty.Fd())
+
+	oldTermStatePtr, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
+	if err != nil {
+		return nil, fmt.Errorf("Can't get terminal settings: %v", err)
+	}
+
+	oldTermios := *oldTermStatePtr
+	newTermios := oldTermios
+	newTermios.Lflag &^= syscall.ECHO
+	newTermios.Lflag |= syscall.ICANON | syscall.ISIG
+	newTermios.Iflag |= syscall.ICRNL
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, &newTermios); err != nil {
+		return nil, fmt.Errorf("Can't set terminal settings: %v", err)
+	}
+
+	quit = make(chan int, 1)
+	go catchTerminate(quit)
+	return
+}
+
+func unlockEcho() error {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if origTermStatePtr == nil {
+		return nil
+	}
+
+	fd := int(tty.Fd())
+
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, origTermStatePtr); err != nil {
+		return fmt.Errorf("Can't set terminal settings: %v", err)
+	}
+
+	origTermStatePtr = nil
+
+	return nil
+}
+
+// listen exit signals and restore terminal state
+func catchTerminate(quit chan int) {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
+	defer signal.Stop(sig)
+	select {
+	case <-quit:
+		unlockEcho()
+	case <-sig:
+		unlockEcho()
+	}
+}

vendor/github.com/cheggaaa/pb/pool.go

@@ -0,0 +1,82 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows
+
+package pb
+
+import (
+	"io"
+	"sync"
+	"time"
+)
+
+// Create and start new pool with given bars
+// You need call pool.Stop() after work
+func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) {
+	pool = new(Pool)
+	if err = pool.start(); err != nil {
+		return
+	}
+	pool.Add(pbs...)
+	return
+}
+
+type Pool struct {
+	Output        io.Writer
+	RefreshRate   time.Duration
+	bars          []*ProgressBar
+	lastBarsCount int
+	quit          chan int
+	m             sync.Mutex
+	finishOnce    sync.Once
+}
+
+// Add progress bars.
+func (p *Pool) Add(pbs ...*ProgressBar) {
+	p.m.Lock()
+	defer p.m.Unlock()
+	for _, bar := range pbs {
+		bar.ManualUpdate = true
+		bar.NotPrint = true
+		bar.Start()
+		p.bars = append(p.bars, bar)
+	}
+}
+
+func (p *Pool) start() (err error) {
+	p.RefreshRate = DefaultRefreshRate
+	quit, err := lockEcho()
+	if err != nil {
+		return
+	}
+	p.quit = make(chan int)
+	go p.writer(quit)
+	return
+}
+
+func (p *Pool) writer(finish chan int) {
+	var first = true
+	for {
+		select {
+		case <-time.After(p.RefreshRate):
+			if p.print(first) {
+				p.print(false)
+				finish <- 1
+				return
+			}
+			first = false
+		case <-p.quit:
+			finish <- 1
+			return
+		}
+	}
+}
+
+// Restore terminal state and close pool
+func (p *Pool) Stop() error {
+	// Wait until one final refresh has passed.
+	time.Sleep(p.RefreshRate)
+
+	p.finishOnce.Do(func() {
+		close(p.quit)
+	})
+	return unlockEcho()
+}

vendor/github.com/cheggaaa/pb/pool_win.go

@@ -0,0 +1,45 @@
+// +build windows
+
+package pb
+
+import (
+	"fmt"
+	"log"
+)
+
+func (p *Pool) print(first bool) bool {
+	p.m.Lock()
+	defer p.m.Unlock()
+	var out string
+	if !first {
+		coords, err := getCursorPos()
+		if err != nil {
+			log.Panic(err)
+		}
+		coords.Y -= int16(p.lastBarsCount)
+		if coords.Y < 0 {
+			coords.Y = 0
+		}
+		coords.X = 0
+
+		err = setCursorPos(coords)
+		if err != nil {
+			log.Panic(err)
+		}
+	}
+	isFinished := true
+	for _, bar := range p.bars {
+		if !bar.IsFinished() {
+			isFinished = false
+		}
+		bar.Update()
+		out += fmt.Sprintf("\r%s\n", bar.String())
+	}
+	if p.Output != nil {
+		fmt.Fprint(p.Output, out)
+	} else {
+		fmt.Print(out)
+	}
+	p.lastBarsCount = len(p.bars)
+	return isFinished
+}

vendor/github.com/cheggaaa/pb/pool_x.go

@@ -0,0 +1,29 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly
+
+package pb
+
+import "fmt"
+
+func (p *Pool) print(first bool) bool {
+	p.m.Lock()
+	defer p.m.Unlock()
+	var out string
+	if !first {
+		out = fmt.Sprintf("\033[%dA", p.lastBarsCount)
+	}
+	isFinished := true
+	for _, bar := range p.bars {
+		if !bar.IsFinished() {
+			isFinished = false
+		}
+		bar.Update()
+		out += fmt.Sprintf("\r%s\n", bar.String())
+	}
+	if p.Output != nil {
+		fmt.Fprint(p.Output, out)
+	} else {
+		fmt.Print(out)
+	}
+	p.lastBarsCount = len(p.bars)
+	return isFinished
+}

vendor/github.com/cheggaaa/pb/reader.go

@@ -0,0 +1,25 @@
+package pb
+
+import (
+	"io"
+)
+
+// It's proxy reader, implement io.Reader
+type Reader struct {
+	io.Reader
+	bar *ProgressBar
+}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+	n, err = r.Reader.Read(p)
+	r.bar.Add(n)
+	return
+}
+
+// Close the reader when it implements io.Closer
+func (r *Reader) Close() (err error) {
+	if closer, ok := r.Reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return
+}

vendor/github.com/cheggaaa/pb/runecount.go

@@ -0,0 +1,17 @@
+package pb
+
+import (
+	"github.com/mattn/go-runewidth"
+	"regexp"
+)
+
+// Finds the control character sequences (like colors)
+var ctrlFinder = regexp.MustCompile("\x1b\x5b[0-9]+\x6d")
+
+func escapeAwareRuneCountInString(s string) int {
+	n := runewidth.StringWidth(s)
+	for _, sm := range ctrlFinder.FindAllString(s, -1) {
+		n -= runewidth.StringWidth(sm)
+	}
+	return n
+}

vendor/github.com/cheggaaa/pb/termios_bsd.go

@@ -0,0 +1,9 @@
+// +build darwin freebsd netbsd openbsd dragonfly
+// +build !appengine
+
+package pb
+
+import "syscall"
+
+const ioctlReadTermios = syscall.TIOCGETA
+const ioctlWriteTermios = syscall.TIOCSETA

vendor/github.com/cheggaaa/pb/termios_sysv.go

@@ -0,0 +1,13 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux solaris
+// +build !appengine
+
+package pb
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETS
+const ioctlWriteTermios = unix.TCSETS